level=info caller=logger.go:50 ts=2024-07-11T19:34:22.12784089Z msg="metrics collector initialized"
W0711 19:34:22.128426       1 client_config.go:618] Neither --kubeconfig nor --master was specified.  Using the inClusterConfig.  This might not work.
level=warn caller=logger.go:55 ts=2024-07-11T19:34:22.140262482Z component=forwarder msg=https://observatorium-api-open-cluster-management-observability.apps.acm-gigantic.rdu2.scalelab.redhat.com/api/metrics/v1/default/api/v1/receive
level=warn caller=logger.go:55 ts=2024-07-11T19:34:22.140293265Z component=forwarder msg="not anonymizing any labels"
level=warn caller=logger.go:55 ts=2024-07-11T19:34:22.157187425Z component=forwarder msg="no certs found in from-ca-file"
W0711 19:34:22.157432       1 client_config.go:618] Neither --kubeconfig nor --master was specified.  Using the inClusterConfig.  This might not work.
level=info caller=logger.go:50 ts=2024-07-11T19:34:22.157721647Z msg="starting metrics collector" from=https://prometheus-k8s.openshift-monitoring.svc:9091 to=https://observatorium-api-open-cluster-management-observability.apps.acm-gigantic.rdu2.scalelab.redhat.com/api/metrics/v1/default/api/v1/receive listen=:8080
level=warn caller=logger.go:55 ts=2024-07-11T19:34:22.157909223Z msg="no certs found in from-ca-file"
level=error caller=logger.go:60 ts=2024-07-11T19:34:23.212608332Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T19:34:23.213440693Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
caller=status.go:110 ts=2024-07-11T19:34:23.216925104Z component=forwarder component=statusclient msg="Updating status of ObservabilityAddon open-cluster-management-addon-observability/observability-addon" type=Degraded status=True reason=Degraded
level=error caller=logger.go:60 ts=2024-07-11T19:34:23.218349042Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:34:23.221608029Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T19:34:53.24554283Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:34:53.250513545Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T19:35:23.254810671Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": dial tcp [fd02::526f]:9091: connect: connection refused"
level=error caller=logger.go:60 ts=2024-07-11T19:35:23.25523368Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": dial tcp [fd02::526f]:9091: connect: connection refused" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:35:23.25692352Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": dial tcp [fd02::526f]:9091: connect: connection refused" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:35:23.260959463Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": dial tcp [fd02::526f]:9091: connect: connection refused"
level=error caller=logger.go:60 ts=2024-07-11T19:35:53.262963342Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": dial tcp [fd02::526f]:9091: connect: connection refused" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:35:53.26456936Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": dial tcp [fd02::526f]:9091: connect: connection refused" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T19:36:24.293036353Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T19:36:24.293711078Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:36:24.298332234Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:36:24.298582159Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T19:36:54.308408858Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:36:54.312880336Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T19:37:24.306730536Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T19:37:24.312623088Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T19:37:24.318516874Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:37:24.322673837Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:37:54.330092904Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:37:54.334478644Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T19:38:24.326396895Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T19:38:24.332789845Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T19:38:24.346121364Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:38:24.351617941Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:38:54.358800779Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:38:54.364167552Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T19:39:24.343794944Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T19:39:24.349510097Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T19:39:24.369326147Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:39:24.374036419Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:39:54.38194959Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:39:54.386375892Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T19:40:24.358162149Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T19:40:24.364602518Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T19:40:24.392440443Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:40:24.397202202Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:40:54.404383924Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:40:54.408891779Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T19:41:24.372187018Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T19:41:24.378178175Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T19:41:24.414091486Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:41:24.418410094Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:41:54.425759842Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:41:54.430384615Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T19:42:24.388860438Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T19:42:24.395099491Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T19:42:24.436484908Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:42:24.441072988Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:42:54.448813449Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:42:54.453571799Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T19:43:24.403398774Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T19:43:24.409230262Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T19:43:24.459587071Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:43:24.464829844Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:43:54.472792165Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:43:54.478096947Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T19:44:24.418665788Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T19:44:24.424994587Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T19:44:24.484658515Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:44:24.489039379Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:44:54.496917287Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:44:54.501839963Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T19:45:24.433400428Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T19:45:24.439765091Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T19:45:24.507550893Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:45:24.511913099Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:45:54.519417667Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:45:54.524467705Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T19:46:24.447627886Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T19:46:24.455158562Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T19:46:24.530086708Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:46:24.534451132Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:46:54.542358963Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:46:54.546976109Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T19:47:24.464207652Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T19:47:24.469323061Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T19:47:24.552589207Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:47:24.557371252Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:47:54.564957595Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:47:54.569861071Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T19:48:24.480404555Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T19:48:24.486600195Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T19:48:24.575212829Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:48:24.579810727Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:48:54.58675974Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:48:54.59183821Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T19:49:24.495554956Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T19:49:24.502032795Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T19:49:24.59855417Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:49:24.603481768Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:49:54.610772099Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:49:54.615507063Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T19:50:24.512662004Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T19:50:24.518490426Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T19:50:24.620913464Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:50:24.625334449Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:50:54.632438217Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:50:54.636979538Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T19:51:24.526134281Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T19:51:24.532299913Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T19:51:24.642698046Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:51:24.647231601Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:51:54.654329623Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:51:54.659040438Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T19:52:24.539593303Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T19:52:24.545026512Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T19:52:24.665056274Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:52:24.669478824Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:52:54.67643304Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:52:54.681137071Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T19:53:24.552956698Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T19:53:24.559177666Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T19:53:24.688730859Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:53:24.693357068Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:53:54.701522021Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:53:54.706197836Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T19:54:24.570852785Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T19:54:24.576879089Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T19:54:24.711840188Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:54:24.71642147Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:54:54.723074034Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:54:54.727453619Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T19:55:24.584780986Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T19:55:24.59042001Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T19:55:24.733345714Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:55:24.738273124Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:55:54.744924665Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:55:54.749592482Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T19:56:24.60080265Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T19:56:24.606884007Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T19:56:24.756926973Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:56:24.763766726Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:56:54.770695776Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:56:54.774941275Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T19:57:24.616605803Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T19:57:24.622622533Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T19:57:24.781883302Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:57:24.787035581Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:57:54.793905914Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:57:54.79829855Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T19:58:24.631209294Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T19:58:24.637006046Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T19:58:24.80534681Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:58:24.810416449Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:58:54.818494461Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:58:54.823944632Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T19:59:24.645381839Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T19:59:24.651187579Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T19:59:24.830298379Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:59:24.834740753Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:59:54.842534771Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T19:59:54.847299603Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:00:24.662617681Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:00:24.668796232Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:00:24.853167156Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:00:24.857565345Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:00:54.865175401Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:00:54.87007654Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:01:24.678231981Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:01:24.686871107Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:01:24.875917028Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:01:24.880477751Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:01:54.887286597Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:01:54.892057422Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:02:24.697445946Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:02:24.703521686Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:02:24.897496229Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:02:24.901651246Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:02:54.90841129Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:02:54.913098049Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:03:24.711950861Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:03:24.719231337Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:03:24.919528999Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:03:24.92684264Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:03:54.934396557Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:03:54.939187215Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:04:24.727201843Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:04:24.733093501Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:04:24.945253182Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:04:24.949990219Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:04:54.958134677Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:04:54.962643961Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:05:24.742854063Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:05:24.74889767Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:05:24.969155482Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:05:24.974229253Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:05:54.982487967Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:05:54.987092291Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:06:24.757925548Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:06:24.763667387Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:06:24.993671569Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:06:24.999139057Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:06:55.00707757Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:06:55.011701801Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:07:24.772108159Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:07:24.778605032Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:07:25.017458578Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:07:25.021969516Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:07:55.030123867Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:07:55.035080676Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:08:24.788102124Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:08:24.793858573Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:08:25.041766453Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:08:25.047295422Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:08:55.055178731Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:08:55.059681489Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:09:24.802721302Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:09:24.811470212Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:09:25.066154344Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:09:25.070789643Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:09:55.078364783Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:09:55.083568546Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:10:24.822074629Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:10:24.829826408Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:10:25.089303133Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:10:25.094559288Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:10:55.102593342Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:10:55.107236408Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:11:24.839307438Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:11:24.845662091Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:11:25.113500201Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:11:25.119688915Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:11:55.126760425Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:11:55.131704147Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:12:24.856055604Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:12:24.862079295Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:12:25.140578136Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:12:25.147308262Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:12:55.15568024Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:12:55.160454083Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:13:24.870188178Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:13:24.875999839Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:13:25.165935324Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:13:25.169947012Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:13:55.177299168Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:13:55.18186015Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:14:24.886839913Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:14:24.892995313Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:14:25.187924076Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:14:25.192705848Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:14:55.199475025Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:14:55.203863781Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:15:24.902327301Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:15:24.908347896Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:15:25.209675909Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:15:25.214095885Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:15:55.220448714Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:15:55.224639001Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:16:24.916699239Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:16:24.922945456Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:16:25.230724425Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:16:25.235514555Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:16:55.242676566Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:16:55.247260652Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:17:24.931200742Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:17:24.938083007Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:17:25.253256928Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:17:25.257883093Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:17:55.264743127Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:17:55.268961286Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:18:24.946969954Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:18:24.952845849Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:18:25.274747485Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:18:25.280244869Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:18:55.288836768Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:18:55.302487797Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:19:24.961719576Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:19:24.967975782Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:19:25.308123954Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:19:25.312737976Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:19:55.31950209Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:19:55.323971233Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:20:24.979649377Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:20:24.986784806Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:20:25.330417103Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:20:25.335124993Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:20:55.3414612Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:20:55.345979583Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:21:24.995129433Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:21:25.001378544Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:21:25.352884344Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:21:25.357428127Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:21:55.365149743Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:21:55.369865195Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:22:25.010479267Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:22:25.016174779Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:22:25.376005008Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:22:25.380633179Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:22:55.388077917Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:22:55.393628632Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:23:25.025329127Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:23:25.033294164Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:23:25.400585673Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:23:25.405285Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:23:55.414403698Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:23:55.418997206Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:24:25.041378648Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:24:25.054038347Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:24:25.424472075Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:24:25.42902555Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:24:55.436792112Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:24:55.441903734Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:25:25.062546362Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:25:25.068823916Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:25:25.448189403Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:25:25.454067046Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:25:55.461153498Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:25:55.465452233Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:26:25.078622092Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:26:25.084589921Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:26:25.470953159Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:26:25.475856303Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:26:55.48381593Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:26:55.488708846Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:27:25.093164208Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:27:25.100589381Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:27:25.49484013Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:27:25.500101225Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:27:55.508617789Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:27:55.513851874Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:28:25.112000345Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:28:25.118550418Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:28:25.520214794Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:28:25.525251443Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:28:55.534112573Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:28:55.539477379Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:29:25.126903806Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:29:25.132976201Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:29:25.546281974Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:29:25.550952695Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:29:55.558087838Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:29:55.562647289Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:30:25.140650692Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:30:25.146948125Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:30:25.56877095Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:30:25.57358973Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:30:55.581200786Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:30:55.58596605Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:31:25.15825082Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:31:25.164310396Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:31:25.591424657Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:31:25.596676136Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:31:55.606028006Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:31:55.612722103Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:32:25.177904611Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:32:25.184138142Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:32:25.618882418Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:32:25.623389935Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:32:55.63095015Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:32:55.635148018Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:33:25.194000121Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:33:25.20166471Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:33:25.641319167Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:33:25.646386464Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:33:55.654704821Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:33:55.659377395Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:34:25.213016273Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:34:25.21947675Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:34:25.669089971Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:34:25.67434468Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:34:55.68125096Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:34:55.68598678Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:35:25.229292599Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:35:25.236760631Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:35:25.692674644Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:35:25.697718501Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:35:55.705912464Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:35:55.710508885Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:36:25.24833358Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:36:25.254097137Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:36:25.716828637Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:36:25.722342276Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:36:55.73001304Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:36:55.734789675Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:37:25.26249095Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:37:25.270819261Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:37:25.740988464Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:37:25.745490814Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:37:55.75313264Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:37:55.758504116Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:38:25.280398348Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:38:25.28639542Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:38:25.764463183Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:38:25.769538371Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:38:55.776398606Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:38:55.781184087Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:39:25.295118388Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:39:25.300980718Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:39:25.786835124Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:39:25.791368624Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:39:55.80007787Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:39:55.805156539Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:40:25.312759072Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:40:25.318834914Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:40:25.811440871Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:40:25.816165291Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:40:55.824248897Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:40:55.82907909Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:41:25.32766455Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:41:25.333608505Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:41:25.835657482Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:41:25.841686441Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:41:55.849879509Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:41:55.855541517Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:42:25.34146819Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:42:25.347283622Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:42:25.862606821Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:42:25.867466927Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:42:55.875170753Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:42:55.880006643Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:43:25.357304631Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:43:25.363770635Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:43:25.886909292Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:43:25.891766692Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:43:55.899649771Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:43:55.904495462Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:44:25.375832637Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:44:25.382078027Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:44:25.910737145Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:44:25.917497075Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:44:55.926061884Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:44:55.931655519Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:45:25.390683564Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:45:25.399796111Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:45:25.938229483Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:45:25.942773299Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:45:55.950571107Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:45:55.955242082Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:46:25.407279206Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:46:25.413389524Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:46:25.962117871Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:46:25.967637525Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:46:55.976723717Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:46:55.983179805Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:47:25.422824745Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:47:25.430267222Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:47:25.98937253Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:47:25.994550431Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:47:56.003062297Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:47:56.007954489Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:48:25.44460985Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:48:25.450430484Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:48:26.014676373Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:48:26.019469055Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:48:56.028323473Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:48:56.032639998Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:49:25.459487565Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:49:25.465904073Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:49:26.038810148Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:49:26.043813944Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:49:56.051600543Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:49:56.056635962Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:50:25.475510994Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:50:25.482498313Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:50:26.063070683Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:50:26.06781787Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:50:56.075876804Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:50:56.080962882Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:51:25.49354566Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:51:25.500619719Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:51:26.086999359Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:51:26.093264204Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:51:56.103445573Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:51:56.11088945Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:52:25.511392241Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:52:25.516972731Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:52:26.117021069Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:52:26.122478059Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:52:56.130261259Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:52:56.135261805Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:53:25.527674376Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:53:25.533588818Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:53:26.140507991Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:53:26.144909777Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:53:56.152306822Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:53:56.157375321Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:54:25.545886604Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:54:25.552038456Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:54:26.164212498Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:54:26.169224317Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:54:56.177194532Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:54:56.18224094Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:55:25.560583487Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:55:25.566898266Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:55:26.189322682Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:55:26.195717591Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:55:56.204370773Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:55:56.2099689Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:56:25.580979508Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:56:25.588088752Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:56:26.216536405Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:56:26.222886816Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:56:56.23869036Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:56:56.248987063Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:57:25.59918781Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:57:25.607143017Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:57:26.255567536Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:57:26.260354196Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:57:56.268860956Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:57:56.274534773Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:58:25.625794513Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:58:25.637801157Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:58:26.280484927Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:58:26.285326108Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:58:56.295334126Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:58:56.300017351Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T20:59:25.64670526Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:59:25.653675813Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T20:59:26.305948597Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:59:26.310292058Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:59:56.318128039Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T20:59:56.323324778Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:00:25.662037675Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:00:25.66866934Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:00:26.329166223Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:00:26.334171379Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:00:56.34206761Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:00:56.347265189Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:01:25.678419995Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:01:25.685331384Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:01:26.355010807Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:01:26.360723299Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:01:56.36832348Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:01:56.372985154Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:02:25.696522319Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:02:25.702726517Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:02:26.378562578Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:02:26.383756266Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:02:56.392897923Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:02:56.398780447Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:03:25.712321248Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:03:25.71886329Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:03:26.404668152Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:03:26.409436753Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:03:56.418179046Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:03:56.424296811Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:04:25.730842119Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:04:25.737313573Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:04:26.430670323Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:04:26.435459737Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:04:56.444639392Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:04:56.450016189Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:05:25.746820077Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:05:25.753665348Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:05:26.456014102Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:05:26.460513148Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:05:56.468615695Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:05:56.473378077Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:06:25.765436946Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:06:25.771702651Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:06:26.479638568Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:06:26.48392907Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:06:56.490931961Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:06:56.495386417Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:07:25.780218579Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:07:25.786177781Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:07:26.502687513Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:07:26.508768117Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:07:56.51663126Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:07:56.521219168Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:08:25.794603492Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:08:25.801327131Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:08:26.527228283Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:08:26.532127117Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:08:56.540377511Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:08:56.545082374Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:09:25.811124875Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:09:25.819404276Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:09:26.551984475Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:09:26.556529655Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:09:56.564775823Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:09:56.56969083Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:10:25.832040592Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:10:25.838422333Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:10:26.576716271Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:10:26.581764273Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:10:56.591580181Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:10:56.600192837Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:11:25.847204157Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:11:25.85338266Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:11:26.606944569Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:11:26.612764825Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:11:56.62083497Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:11:56.626142122Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:12:25.865094238Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:12:25.870674Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:12:26.632621868Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:12:26.637376967Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:12:56.644559489Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:12:56.649731105Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:13:25.881963721Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:13:25.889354133Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:13:26.659840074Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:13:26.669256146Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:13:56.678098547Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:13:56.683484608Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:14:25.900746883Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:14:25.906387686Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:14:26.689859676Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:14:26.695617719Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:14:56.702898994Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:14:56.708361887Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:15:25.917278844Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:15:25.924171827Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:15:26.715585572Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:15:26.726593806Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:15:56.735019072Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:15:56.74045759Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:16:25.935252133Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:16:25.941638078Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:16:26.746702004Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:16:26.751899644Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:16:56.761543266Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:16:56.768807282Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:17:25.950139804Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:17:25.956601572Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:17:26.77439161Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:17:26.779494069Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:17:56.787549363Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:17:56.793401341Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:18:25.970714016Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:18:25.977700352Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:18:26.800500528Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:18:26.805834052Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:18:56.815651322Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:18:56.821672824Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:19:25.98930874Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:19:25.997119525Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:19:26.829438629Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:19:26.835490995Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:19:56.843795964Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:19:56.849037196Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:20:26.010455795Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:20:26.016826078Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:20:26.854597965Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:20:26.859113398Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:20:56.867455888Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:20:56.872235418Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:21:26.025692777Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:21:26.032223807Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:21:26.878875775Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:21:26.883970811Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:21:56.893568976Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:21:56.898563819Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:22:26.040620824Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:22:26.048014867Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:22:26.903880219Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:22:26.908483883Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:22:56.917036989Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:22:56.922440229Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:23:26.056809064Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:23:26.063717967Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:23:26.928769647Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:23:26.935663273Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:23:56.943439964Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:23:56.94836155Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:24:26.075854768Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:24:26.082650085Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:24:26.955888846Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:24:26.961093316Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:24:56.969728991Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:24:56.974774904Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:25:26.091532889Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:25:26.098025944Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:25:26.981640581Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:25:26.98746314Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:25:56.995953783Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:25:57.000429741Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:26:26.109013599Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:26:26.11700557Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:26:27.006630714Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:26:27.012379952Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:26:57.019870456Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:26:57.024982903Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:27:26.126067824Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:27:26.132295955Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:27:27.031119848Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:27:27.036805743Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:27:57.044406424Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:27:57.04953166Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:28:26.144484212Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:28:26.150519499Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:28:27.054759054Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:28:27.059324258Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:28:57.066729286Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:28:57.071468537Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:29:26.160332984Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:29:26.166659244Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:29:27.07811692Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:29:27.083520711Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:29:57.092716438Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:29:57.099381987Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:30:26.180483555Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:30:26.187268132Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:30:27.106263916Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:30:27.111259901Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:30:57.118732153Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:30:57.124038902Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:31:26.198561761Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:31:26.205425509Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:31:27.131423336Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:31:27.137197721Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:31:57.146029831Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:31:57.151266415Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:32:26.216025722Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:32:26.222255942Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:32:27.156920017Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:32:27.161865204Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:32:57.169308666Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:32:57.174295729Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:33:26.233247893Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:33:26.239433226Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:33:27.181653202Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:33:27.186812073Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:33:57.200687815Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:33:57.208424679Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:34:26.250952288Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:34:26.25747206Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:34:27.214309889Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:34:27.220437311Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:34:57.228776184Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:34:57.233999319Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:35:26.266734176Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:35:26.273794537Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:35:27.241587381Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:35:27.246800256Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:35:57.253959734Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:35:57.26057969Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:36:26.281387874Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:36:26.28737759Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:36:27.266626731Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:36:27.271998764Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:36:57.280307045Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:36:57.285900997Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:37:26.298355134Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:37:26.306253634Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:37:27.29256405Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:37:27.297653133Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:37:57.30581874Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:37:57.310694077Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:38:26.317345768Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:38:26.32418852Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:38:27.316033884Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:38:27.321147537Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:38:57.32838347Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:38:57.333492804Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:39:26.334847137Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:39:26.341630529Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:39:27.339212633Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:39:27.343958322Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:39:57.353505452Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:39:57.359719537Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:40:26.354543769Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:40:26.359758212Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:40:27.366896203Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:40:27.372281162Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:40:57.38168401Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:40:57.388360251Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:41:26.370179149Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:41:26.376203549Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:41:27.395439153Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:41:27.40092853Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:41:57.409109539Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:41:57.414101306Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:42:26.391414985Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:42:26.398099532Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:42:27.421199757Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:42:27.426076489Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:42:57.434649518Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:42:57.440827358Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:43:26.408157119Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:43:26.417695549Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:43:27.446797003Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:43:27.451658299Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:43:57.460426795Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:43:57.466788225Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:44:26.429441679Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:44:26.437024835Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:44:27.473701105Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:44:27.478807742Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:44:57.490087335Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:44:57.497274327Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:45:26.446884149Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:45:26.456250398Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:45:27.504415954Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:45:27.510457879Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:45:57.51810376Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:45:57.522856136Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:46:26.467032366Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:46:26.47326301Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:46:27.528876874Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:46:27.533445509Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:46:57.541004799Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:46:57.545659247Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:47:26.482209621Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:47:26.488751458Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:47:27.551823816Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:47:27.556947299Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:47:57.564506978Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:47:57.569229999Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:48:26.502090638Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:48:26.507862195Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:48:27.575579979Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:48:27.580426563Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:48:57.587546924Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:48:57.592539485Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:49:26.516601973Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:49:26.522927727Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:49:27.599430452Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:49:27.606206812Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:49:57.615114813Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:49:57.621241019Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:50:26.535449334Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:50:26.541709052Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:50:27.62873124Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:50:27.633545539Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:50:57.643284697Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:50:57.6492567Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:51:26.550300491Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:51:26.556683096Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:51:27.656297078Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:51:27.661949437Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:51:57.669937055Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:51:57.674882029Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:52:26.568633187Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:52:26.5751652Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:52:27.681602718Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:52:27.686375597Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:52:57.694412741Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:52:57.699516306Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:53:26.583683913Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:53:26.590954264Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:53:27.705656297Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:53:27.710842688Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:53:57.718921627Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:53:57.725104033Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:54:26.605421563Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:54:26.611010792Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:54:27.730240377Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:54:27.735526576Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:54:57.743913793Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:54:57.748855143Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:55:26.623532399Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:55:26.629983454Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:55:27.756429732Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:55:27.763498458Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:55:57.7713446Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:55:57.77651711Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:56:26.639428447Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:56:26.645456325Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:56:27.783512758Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:56:27.788618757Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:56:57.797094657Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:56:57.802082819Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:57:26.657595332Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:57:26.66547886Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:57:27.809746198Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:57:27.816210859Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:57:57.827097071Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:57:57.834935206Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:58:26.683209788Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:58:26.692644004Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:58:27.843682973Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:58:27.850961976Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:58:57.858710295Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:58:57.863741123Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T21:59:26.704690554Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:59:26.711304934Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T21:59:27.872789903Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:59:27.880169679Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:59:57.888780225Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T21:59:57.894263146Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:00:26.723912406Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:00:26.731385951Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:00:27.900589173Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:00:27.905802422Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:00:57.913931786Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:00:57.919228379Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:01:26.751731183Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:01:26.75941137Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:01:27.926240376Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:01:27.93167599Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:01:57.942139685Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:01:57.949035936Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:02:26.77100637Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:02:26.77958046Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:02:27.958016945Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:02:27.963488748Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:02:57.972010015Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:02:57.978132386Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:03:26.790079646Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:03:26.796569634Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:03:27.984977703Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:03:27.990650999Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:03:57.99915336Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:03:58.003988722Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:04:26.810087027Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:04:26.817367836Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:04:28.01477666Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:04:28.022694917Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:04:58.029831861Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:04:58.034591385Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:05:26.827451751Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:05:26.834930158Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:05:28.040313263Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:05:28.045453945Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:05:58.054479172Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:05:58.059980609Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:06:26.849814939Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:06:26.856998288Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:06:28.066145855Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:06:28.07229143Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:06:58.080581977Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:06:58.085745469Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:07:26.870084535Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:07:26.878023361Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:07:28.096236556Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:07:28.104556545Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:07:58.112460313Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:07:58.117949741Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:08:26.892335859Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:08:26.898544091Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:08:28.124256098Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:08:28.130260115Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:08:58.137461218Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:08:58.142299555Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:09:26.912383156Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:09:26.918937604Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:09:28.148493246Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:09:28.154500774Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:09:58.163190608Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:09:58.1686537Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:10:26.931276391Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:10:26.938459112Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:10:28.174416871Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:10:28.179160942Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:10:58.188663793Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:10:58.194228809Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:11:26.949405761Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:11:26.956877647Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:11:28.200594691Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:11:28.205270297Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:11:58.213472971Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:11:58.218810615Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:12:26.969173134Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:12:26.976985741Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:12:28.225014105Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:12:28.230614652Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:12:58.238166822Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:12:58.242924922Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:13:26.98882754Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:13:26.9957996Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:13:28.248848534Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:13:28.254152791Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:13:58.263904903Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:13:58.269304946Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:14:27.010363635Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:14:27.016783914Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:14:28.275493382Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:14:28.280674785Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:14:58.290208933Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:14:58.295810319Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:15:27.02575935Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:15:27.033218528Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:15:28.303785575Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:15:28.311072964Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:15:58.3208918Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:15:58.328658323Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:16:27.041745178Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:16:27.047652864Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:16:28.33519073Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:16:28.341218812Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:16:58.355184087Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:16:58.362314774Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:17:27.057584483Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:17:27.066356502Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:17:28.368952657Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:17:28.373698845Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:17:58.382061031Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:17:58.388274894Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:18:27.085536655Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:18:27.092450686Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:18:28.396443106Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:18:28.403400133Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:18:58.411430532Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:18:58.416744541Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:19:27.105808983Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:19:27.112915479Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:19:28.423768495Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:19:28.431881112Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:19:58.455930697Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:19:58.471984727Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:20:27.129723161Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:20:27.137858435Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:20:28.478237381Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:20:28.483545543Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:20:58.491612988Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:20:58.496589062Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:21:27.148562149Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:21:27.156268294Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:21:28.504321375Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:21:28.510724107Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:21:58.523678579Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:21:58.534737582Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:22:27.173171844Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:22:27.181002472Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:22:28.541226648Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:22:28.547793886Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:22:58.574878801Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:22:58.59385029Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:23:27.192056666Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:23:27.19892536Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:23:28.602021641Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:23:28.608520711Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:23:58.618234224Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:23:58.623517246Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:24:27.213656208Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:24:27.220383267Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:24:28.62970382Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:24:28.634637087Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:24:58.642482315Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:24:58.64777131Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:25:27.229839971Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:25:27.236410445Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:25:28.653988838Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:25:28.659303054Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:25:58.667875271Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:25:58.672749305Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:26:27.250211816Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:26:27.257758977Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:26:28.681432048Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:26:28.689362219Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:26:58.698719496Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:26:58.704068377Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:27:27.267650654Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:27:27.274312596Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:27:28.710936885Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:27:28.716393759Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:27:58.724971399Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:27:58.730575328Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:28:27.290258307Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:28:27.296593653Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:28:28.740172566Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:28:28.74672447Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:28:58.757353738Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:28:58.763538676Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:29:27.307238515Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:29:27.314331376Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:29:28.770852218Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:29:28.775916932Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:29:58.785054528Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:29:58.791114821Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:30:27.328459416Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:30:27.335251662Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:30:28.798480324Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:30:28.804496011Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:30:58.814149144Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:30:58.819464477Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:31:27.346815775Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:31:27.352876764Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:31:28.825671086Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:31:28.83067664Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:31:58.839553155Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:31:58.844612192Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:32:27.366157615Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:32:27.37272089Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:32:28.850786329Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:32:28.85592694Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:32:58.864969538Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:32:58.86996188Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:33:27.386182497Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:33:27.392498084Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:33:28.87717405Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:33:28.882545026Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:33:58.890383813Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:33:58.895458119Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:34:27.403225788Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:34:27.410263385Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:34:28.901532496Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:34:28.906950396Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:34:58.915280102Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:34:58.920646305Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:35:27.421412596Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:35:27.429911744Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:35:28.93020183Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:35:28.937026836Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:35:58.945331374Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:35:58.95108226Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:36:27.442984699Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:36:27.45071133Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:36:28.957955026Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:36:28.963777531Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:36:58.972831562Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:36:58.977954476Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:37:27.46021526Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:37:27.470516575Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:37:28.984589878Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:37:28.990092321Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:37:59.001572657Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:37:59.008467409Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:38:27.481230629Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:38:27.487669149Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:38:29.021028731Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:38:29.026663524Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:38:59.034887829Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:38:59.040914341Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:39:27.511278336Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:39:27.517182759Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:39:29.047598718Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:39:29.053028555Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:39:59.061695212Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:39:59.067021262Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:40:27.541819505Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:40:27.54942631Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:40:29.075592092Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:40:29.082017295Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:40:59.094032378Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:40:59.103780186Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:41:27.560740587Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:41:27.568687214Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:41:29.111378409Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:41:29.117162316Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:41:59.125135426Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:41:59.131305638Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:42:27.583033753Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:42:27.5899551Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:42:29.138285869Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:42:29.143931619Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:42:59.152696727Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:42:59.157468852Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:43:27.601731163Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:43:27.609363692Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:43:29.166614443Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:43:29.174830996Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:43:59.182654216Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:43:59.189236473Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:44:27.625385Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:44:27.632589902Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:44:29.196821423Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:44:29.202287826Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:44:59.210655578Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:44:59.215518769Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:45:27.642998206Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:45:27.650306469Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:45:29.222749919Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:45:29.227940739Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:45:59.238081239Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:45:59.245679603Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:46:27.665630529Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:46:27.672337985Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:46:29.252868816Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:46:29.257954155Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:46:59.266794603Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:46:59.272467925Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:47:27.683111037Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:47:27.692410663Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:47:29.278513809Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:47:29.284559669Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:47:59.293560414Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:47:59.299006798Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:48:27.711315166Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:48:27.71775753Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:48:29.308631595Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:48:29.314224235Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:48:59.321807201Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:48:59.329266095Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:49:27.727701573Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:49:27.736248539Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:49:29.336760317Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:49:29.341933383Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:49:59.350637225Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:49:59.355357083Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:50:27.751664865Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:50:27.758517803Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:50:29.361950847Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:50:29.367407245Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:50:59.375649372Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:50:59.381228332Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:51:27.769988008Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:51:27.777478542Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:51:29.388565767Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:51:29.394035571Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:51:59.401343678Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:51:59.406618486Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:52:27.792287905Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:52:27.798904489Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:52:29.412749263Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:52:29.417860193Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:52:59.427481383Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:52:59.433594881Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:53:27.809763779Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:53:27.817381426Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:53:29.440833632Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:53:29.44536106Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:53:59.454947745Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:53:59.460848546Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:54:27.83051432Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:54:27.836277067Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:54:29.466678916Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:54:29.471968712Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:54:59.48036064Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:54:59.486399947Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:55:27.844863465Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:55:27.851379113Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:55:29.493595595Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:55:29.499021539Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:55:59.50785547Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:55:59.513341505Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:56:27.866264817Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:56:27.873246155Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:56:29.519063587Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:56:29.524127977Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:56:59.532814759Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:56:59.538064577Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:57:27.883581405Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:57:27.89092574Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:57:29.545431328Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:57:29.550940122Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:57:59.560513437Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:57:59.566390452Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:58:27.906420599Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:58:27.914281273Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:58:29.574405044Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:58:29.580556795Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:58:59.589409054Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:58:59.594563101Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T22:59:27.925596155Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:59:27.933724928Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T22:59:29.601342761Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:59:29.606990458Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:59:59.617104743Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T22:59:59.622720807Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:00:27.944588527Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:00:27.952505586Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:00:29.629349645Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:00:29.635186631Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:00:59.644200755Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:00:59.649848391Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:01:27.961535073Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:01:27.968581848Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:01:29.656626597Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:01:29.662713061Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:01:59.672080434Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:01:59.678253917Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:02:27.985214139Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:02:27.993572349Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:02:29.685264492Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:02:29.690612838Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:02:59.702549017Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:02:59.709397741Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:03:28.007912784Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:03:28.016555399Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:03:29.717187946Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:03:29.723312742Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:03:59.73301415Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:03:59.739579517Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:04:28.032247106Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:04:28.038923157Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:04:29.746525301Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:04:29.752082801Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:04:59.761908618Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:04:59.76774941Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:05:28.048714679Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:05:28.055806583Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:05:29.774328524Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:05:29.779509398Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:05:59.788746913Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:05:59.796061061Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:06:28.072908976Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:06:28.084555893Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:06:29.804541313Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:06:29.809820115Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:06:59.821711473Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:06:59.827676003Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:07:28.100593331Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:07:28.111228961Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:07:29.834681986Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:07:29.840513254Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:07:59.850710352Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:07:59.856229219Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:08:28.13296059Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:08:28.14100688Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:08:29.864505739Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:08:29.869819266Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:08:59.879926848Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:08:59.885918033Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:09:28.157319356Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:09:28.163720972Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:09:29.892823429Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:09:29.898302815Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:09:59.907237482Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:09:59.912904983Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:10:28.178836267Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:10:28.185798342Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:10:29.919723303Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:10:29.92508186Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:10:59.936414961Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:10:59.942784335Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:11:28.201481442Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:11:28.211008735Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:11:29.953675996Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:11:29.963272758Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:11:59.973011756Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:11:59.978469071Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:12:28.226468881Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:12:28.233144722Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:12:29.985822142Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:12:29.99189617Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:13:00.003273497Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:13:00.009571054Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:13:28.244842986Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:13:28.252038663Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:13:30.018438479Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:13:30.025063184Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:14:00.037320186Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:14:00.043896947Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:14:28.267516008Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:14:28.275877942Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:14:30.052457056Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:14:30.058157672Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:15:00.069377864Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:15:00.075848933Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:15:28.297378091Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:15:28.309262289Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:15:30.084725193Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:15:30.092238206Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:16:00.102996849Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:16:00.109228972Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:16:28.325820812Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:16:28.3338226Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:16:30.117030319Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:16:30.123797235Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:17:00.134649404Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:17:00.141756984Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:17:28.35835051Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:17:28.367510252Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:17:30.150840957Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:17:30.156819599Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:18:00.169519124Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:18:00.176752765Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:18:28.3837831Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:18:28.391961848Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:18:30.187287669Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:18:30.193680686Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:19:00.202782971Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:19:00.210727506Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:19:28.403373276Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:19:28.410203247Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:19:30.218249719Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:19:30.224108256Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:20:00.234696784Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:20:00.24246549Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:20:28.421519239Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:20:28.430820106Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:20:30.250691769Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:20:30.258342686Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:21:00.269896533Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:21:00.275310239Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:21:28.442460089Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:21:28.450026011Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:21:30.282999458Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:21:30.288917407Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:22:00.299779293Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:22:00.305346388Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:22:28.466071318Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:22:28.473501766Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:22:30.313846636Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:22:30.320590989Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:23:00.328460358Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:23:00.334350304Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:23:28.498790807Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:23:28.506720455Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:23:30.343971769Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:23:30.350845316Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:24:00.358862364Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:24:00.363986818Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:24:28.517720693Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:24:28.525579077Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:24:30.373562075Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:24:30.382375464Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:25:00.391241965Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:25:00.396495271Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:25:28.540124517Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:25:28.547764908Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:25:30.404417111Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:25:30.40968674Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:26:00.453519289Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:26:00.468799784Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:26:28.564757741Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:26:28.574144335Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:26:30.478877087Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:26:30.485232403Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:27:00.501336795Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:27:00.509311987Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:27:28.589760325Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:27:28.598522131Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:27:30.517070206Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:27:30.523369288Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:28:00.534959449Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:28:00.541426799Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:28:28.628596414Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:28:28.636709136Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:28:30.549947356Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:28:30.556190418Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:29:00.566351321Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:29:00.573116213Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:29:28.649277989Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:29:28.65742644Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:29:30.582527758Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:29:30.589290413Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:30:00.603915119Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:30:00.616201505Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:30:28.673875708Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:30:28.68155281Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:30:30.625958368Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:30:30.633400477Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:31:00.64397286Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:31:00.649951904Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:31:28.694386765Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:31:28.702950526Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:31:30.660206653Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:31:30.670685027Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:32:00.69049696Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:32:00.702764332Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:32:28.716032286Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:32:28.725254104Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:32:30.709886079Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:32:30.715482562Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:33:00.724412848Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:33:00.730345726Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:33:28.737379108Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:33:28.744431897Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:33:30.737147273Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:33:30.742202822Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:34:00.752219795Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:34:00.75871312Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:34:28.758989706Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:34:28.767367384Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:34:30.76735007Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:34:30.772859666Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:35:00.782734713Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:35:00.788742996Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:35:28.781615645Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:35:28.794405455Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:35:30.796680499Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:35:30.802788473Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:36:00.812590871Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:36:00.819129214Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:36:28.807196725Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:36:28.815445742Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:36:30.827208626Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:36:30.832675333Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:37:00.842177766Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:37:00.848475679Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:37:28.829251436Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:37:28.836669851Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:37:30.857728682Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:37:30.86464171Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:38:00.873545204Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:38:00.879480808Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:38:28.855868252Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:38:28.864979218Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:38:30.886025502Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:38:30.891607482Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:39:00.900292174Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:39:00.906338572Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:39:28.877922271Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:39:28.884964941Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:39:30.914672307Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:39:30.920122733Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:40:00.934439982Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:40:00.942559264Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:40:28.904839368Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:40:28.91255415Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:40:30.95415794Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:40:30.961129763Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:41:00.971487861Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:41:00.978333809Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:41:28.923963528Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:41:28.931280599Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:41:30.986735977Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:41:30.993198883Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:42:01.002491345Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:42:01.007507319Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:42:28.947580063Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:42:28.954984864Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:42:31.015121765Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:42:31.020636586Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:43:01.030496765Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:43:01.036743986Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:43:28.967000371Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:43:28.974367641Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:43:31.044581607Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:43:31.051665208Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:44:01.062312155Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:44:01.068501553Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:44:28.991161331Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:44:28.99870666Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:44:31.079256583Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:44:31.086274019Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:45:01.095626466Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:45:01.10094395Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:45:29.010655797Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:45:29.017700306Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:45:31.111711908Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:45:31.11842133Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:46:01.126935968Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:46:01.133343199Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:46:29.034396564Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:46:29.045784357Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:46:31.14259314Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:46:31.150481117Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:47:01.162335253Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:47:01.169904537Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:47:29.055463149Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:47:29.062033784Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:47:31.178337805Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:47:31.183755428Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:48:01.193870172Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:48:01.199913649Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:48:29.078246238Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:48:29.086871319Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:48:31.207757131Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:48:31.213965262Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:49:01.22533592Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:49:01.231670634Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:49:29.104685021Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:49:29.116202996Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:49:31.240995866Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:49:31.248165457Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:50:01.261443372Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:50:01.268802901Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:50:29.128506196Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:50:29.135375557Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:50:31.277938702Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:50:31.283811035Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:51:01.294656061Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:51:01.301761453Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:51:29.145098976Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:51:29.152729954Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:51:31.309896856Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:51:31.315200423Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:52:01.324914412Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:52:01.330358295Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:52:29.1697894Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:52:29.177167905Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:52:31.336939534Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:52:31.342888329Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:53:01.353454316Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:53:01.359632038Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:53:29.194170722Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:53:29.206403983Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:53:31.367862786Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:53:31.375884242Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:54:01.385618044Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:54:01.391642003Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:54:29.226164073Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:54:29.234209694Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:54:31.402122133Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:54:31.409544104Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:55:01.421799328Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:55:01.428020293Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:55:29.244587723Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:55:29.251587374Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:55:31.436238239Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:55:31.442189044Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:56:01.451552986Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:56:01.45877025Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:56:29.262550546Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:56:29.269978679Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:56:31.466783457Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:56:31.472908978Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:57:01.486635274Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:57:01.502525839Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:57:29.280796517Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:57:29.290584584Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:57:31.512149896Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:57:31.518254238Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:58:01.532317092Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:58:01.539021127Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:58:29.3097619Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:58:29.3181826Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:58:31.548318345Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:58:31.555322996Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:59:01.565812171Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:59:01.571297577Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-11T23:59:29.328217257Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:59:29.334884471Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-11T23:59:31.579786268Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-11T23:59:31.587678118Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:00:01.602172595Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:00:01.609189543Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:00:29.355709758Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:00:29.364025965Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:00:31.618245185Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:00:31.625631786Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:01:01.636917175Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:01:01.64469855Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:01:29.376296887Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:01:29.384536759Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:01:31.651404713Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:01:31.657918178Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:02:01.670001292Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:02:01.676853705Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:02:29.400640938Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:02:29.408108094Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:02:31.685993758Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:02:31.692638096Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:03:01.702805556Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:03:01.708958354Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:03:29.418550373Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:03:29.42536477Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:03:31.717935858Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:03:31.723883886Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:04:01.733569104Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:04:01.739394864Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:04:29.443284325Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:04:29.451687867Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:04:31.74944663Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:04:31.756065528Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:05:01.771889192Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:05:01.78163407Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:05:29.463402956Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:05:29.470310447Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:05:31.788953952Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:05:31.794230684Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:06:01.804427205Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:06:01.810744565Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:06:29.489825491Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:06:29.496811182Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:06:31.818522909Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:06:31.82450098Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:07:01.834505566Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:07:01.839780948Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:07:29.509316114Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:07:29.516526302Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:07:31.84712128Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:07:31.853678122Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:08:01.863260899Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:08:01.869714095Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:08:29.534517021Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:08:29.541975648Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:08:31.880034727Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:08:31.88632177Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:09:01.895966393Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:09:01.901926754Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:09:29.552931148Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:09:29.560343948Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:09:31.910262975Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:09:31.916198309Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:10:01.928270467Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:10:01.936133506Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:10:29.578908093Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:10:29.58611773Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:10:31.943929284Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:10:31.949645033Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:11:01.958355108Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:11:01.964294344Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:11:29.598893981Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:11:29.605782174Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:11:31.976956954Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:11:31.986866486Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:12:01.994792477Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:12:02.0002814Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:12:29.622023249Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:12:29.630569144Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:12:32.012692372Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:12:32.023838256Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:13:02.032796061Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:13:02.039503371Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:13:29.641950301Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:13:29.649309184Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:13:32.051556816Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:13:32.058086805Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:14:02.069310664Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:14:02.075671886Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:14:29.6668234Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:14:29.675421746Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:14:32.085663155Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:14:32.093908839Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:15:02.122820456Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:15:02.139483351Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:15:29.688204006Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:15:29.695516571Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:15:32.147569035Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:15:32.152607879Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:16:02.161856576Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:16:02.168578472Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:16:29.712707363Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:16:29.720918789Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:16:32.178727212Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:16:32.186862862Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:17:02.196330835Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:17:02.202140592Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:17:29.733821621Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:17:29.741037295Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:17:32.208927312Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:17:32.214756243Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:18:02.227342116Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:18:02.234145145Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:18:29.758684646Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:18:29.76652851Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:18:32.242410873Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:18:32.248204531Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:19:02.257809162Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:19:02.263300585Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:19:29.778122832Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:19:29.785491878Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:19:32.272952242Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:19:32.278612011Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:20:02.287760773Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:20:02.295230263Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:20:29.801364967Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:20:29.808036579Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:20:32.302225723Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:20:32.307590218Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:21:02.315707433Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:21:02.32231346Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:21:29.819342458Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:21:29.826305555Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:21:32.334766169Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:21:32.341497213Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:22:02.351721411Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:22:02.357026445Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:22:29.843823945Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:22:29.850563111Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:22:32.364894164Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:22:32.370741264Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:23:02.380955682Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:23:02.386710258Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:23:29.861212421Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:23:29.868565294Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:23:32.397301923Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:23:32.405274525Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:24:02.415494547Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:24:02.422092518Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:24:29.887745425Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:24:29.894393092Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:24:32.431419289Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:24:32.438665212Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:25:02.450963322Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:25:02.459940247Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:25:29.904697761Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:25:29.911611646Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:25:32.469754152Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:25:32.478333067Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:26:02.49234797Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:26:02.501867911Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:26:29.928904391Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:26:29.937499153Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:26:32.512617964Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:26:32.517839151Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:27:02.530501946Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:27:02.5381644Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:27:29.95165681Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:27:29.95998296Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:27:32.545592346Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:27:32.55172286Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:28:02.562372144Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:28:02.570251705Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:28:29.976279491Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:28:29.982860466Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:28:32.578412492Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:28:32.584473766Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:29:02.595001457Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:29:02.601300197Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:29:29.994749828Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:29:30.004583035Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:29:32.60975101Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:29:32.615855152Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:30:02.627910145Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:30:02.636692653Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:30:30.019656493Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:30:30.025846238Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:30:32.644026718Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:30:32.651186449Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:31:02.65998794Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:31:02.665828698Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:31:30.038148621Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:31:30.045235696Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:31:32.675372389Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:31:32.681152562Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:32:02.689323692Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:32:02.695296277Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:32:30.060104679Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:32:30.067052951Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:32:32.705098545Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:32:32.712838244Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:33:02.722575935Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:33:02.728594046Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:33:30.081397702Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:33:30.08954553Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:33:32.736653611Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:33:32.742287054Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:34:02.752733877Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:34:02.758452201Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:34:30.110707206Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:34:30.118471901Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:34:32.768708486Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:34:32.774071212Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:35:02.784500534Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:35:02.79207291Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:35:30.129778391Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:35:30.136368474Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:35:32.801660275Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:35:32.807597958Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:36:02.818500285Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:36:02.823695946Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:36:30.155903495Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:36:30.165023842Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:36:32.832696748Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:36:32.838260857Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:37:02.851182428Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:37:02.858385052Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:37:30.175015827Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:37:30.182384166Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:37:32.872072036Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:37:32.880632832Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:38:02.891055724Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:38:02.897197361Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:38:30.200941435Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:38:30.20820552Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:38:32.905892766Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:38:32.912593906Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:39:02.92131687Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:39:02.927463104Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:39:30.217841744Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:39:30.224256699Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:39:32.937290483Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:39:32.944333402Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:40:02.95642049Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:40:02.962738009Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:40:30.233319292Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:40:30.240183371Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:40:32.968697327Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:40:32.97472696Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:41:02.985174593Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:41:02.990636975Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:41:30.250239495Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:41:30.257311836Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:41:32.998789425Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:41:33.004331995Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:42:03.016557983Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:42:03.024330216Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:42:30.27664232Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:42:30.283993176Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:42:33.03274541Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:42:33.038849697Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:43:03.051348246Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:43:03.060314764Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:43:30.297884574Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:43:30.304925232Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:43:33.070925402Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:43:33.079928608Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:44:03.090954707Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:44:03.096677264Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:44:30.322986872Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:44:30.330818983Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:44:33.106445644Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:44:33.112248724Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:45:03.121756967Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:45:03.127585875Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:45:30.342688926Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:45:30.350176853Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:45:33.134361077Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:45:33.140527612Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:46:03.152176208Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:46:03.158357304Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:46:30.367257152Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:46:30.375998106Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:46:33.168197434Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:46:33.174867043Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:47:03.185242392Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:47:03.191479098Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:47:30.38820706Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:47:30.395588915Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:47:33.198824444Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:47:33.204522773Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:48:03.215935027Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:48:03.223263672Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:48:30.407460311Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:48:30.415522547Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:48:33.232258303Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:48:33.238385471Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:49:03.247460724Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:49:03.252811927Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:49:30.429744315Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:49:30.437781157Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:49:33.262092217Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:49:33.268291221Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:50:03.278201115Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:50:03.283674822Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:50:30.456696044Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:50:30.463718685Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:50:33.293826352Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:50:33.301624679Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:51:03.311107939Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:51:03.3160378Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:51:30.476012516Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:51:30.483575678Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:51:33.325544031Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:51:33.333002102Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:52:03.346754044Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:52:03.35638754Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:52:30.500419707Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:52:30.507317677Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:52:33.365532094Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:52:33.370704154Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:53:03.379496117Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:53:03.385324632Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:53:30.518350581Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:53:30.525249192Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:53:33.39436205Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:53:33.402011992Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:54:03.412805011Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:54:03.418639724Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:54:30.544003448Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:54:30.550864169Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:54:33.425076124Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:54:33.430823998Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:55:03.443813256Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:55:03.459427133Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:55:30.562396543Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:55:30.569896904Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:55:33.467722399Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:55:33.473793868Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:56:03.484991813Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:56:03.491911723Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:56:30.587753683Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:56:30.595914725Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:56:33.500314796Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:56:33.506692867Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:57:03.517911498Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:57:03.525605735Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:57:30.607534031Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:57:30.616376605Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:57:33.536089613Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:57:33.543822776Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:58:03.553802233Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:58:03.55926294Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:58:30.638038783Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:58:30.6464647Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:58:33.568838714Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:58:33.577627358Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:59:03.593160325Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:59:03.600167855Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T00:59:30.658455252Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:59:30.665640065Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T00:59:33.612507222Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T00:59:33.619832277Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:00:03.633655741Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:00:03.641276385Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:00:30.682183229Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:00:30.690210516Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:00:33.650425372Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:00:33.656158172Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:01:03.668317384Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:01:03.674112991Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:01:30.701855295Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:01:30.708200592Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:01:33.681317422Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:01:33.685998027Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:02:03.696655023Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:02:03.70262469Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:02:30.726709329Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:02:30.734090859Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:02:33.712798752Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:02:33.719460999Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:03:03.731353105Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:03:03.737187227Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:03:30.745150648Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:03:30.752019196Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:03:33.749225046Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:03:33.756896172Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:04:03.768889115Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:04:03.776477063Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:04:30.771267729Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:04:30.779376816Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:04:33.785615137Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:04:33.791843225Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:05:03.802105697Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:05:03.80949086Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:05:30.790778151Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:05:30.799592731Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:05:33.820466628Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:05:33.827534972Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:06:03.838152433Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:06:03.845258611Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:06:30.819471205Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:06:30.82727398Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:06:33.854581295Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:06:33.859895472Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:07:03.867843252Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:07:03.873554308Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:07:30.840882658Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:07:30.848343488Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:07:33.883002675Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:07:33.889138348Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:08:03.898033107Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:08:03.904449105Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:08:30.868115227Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:08:30.875690907Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:08:33.912931279Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:08:33.918537078Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:09:03.930755403Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:09:03.93697168Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:09:30.887824262Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:09:30.89586941Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:09:33.94486732Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:09:33.950008938Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:10:03.961950106Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:10:03.968903227Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:10:30.909490266Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:10:30.917008572Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:10:33.977102713Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:10:33.98288243Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:11:03.997524964Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:11:04.006111494Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:11:30.92972364Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:11:30.937278012Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:11:34.017660155Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:11:34.024332052Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:12:04.033636873Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:12:04.039403615Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:12:30.953704435Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:12:30.960764786Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:12:34.047583278Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:12:34.052713686Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:13:04.063475381Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:13:04.068881963Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:13:30.973986616Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:13:30.982550241Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:13:34.078893576Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:13:34.085576457Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:14:04.094902214Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:14:04.100825653Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:14:31.000934156Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:14:31.008980516Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:14:34.109537804Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:14:34.115407137Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:15:04.130061922Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:15:04.136835833Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:15:31.021456425Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:15:31.028996731Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:15:34.145587706Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:15:34.151584067Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:16:04.162172533Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:16:04.167659969Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:16:31.047566459Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:16:31.05484848Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:16:34.17766814Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:16:34.185478558Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:17:04.199801421Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:17:04.207018849Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:17:31.06845841Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:17:31.075561402Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:17:34.21588436Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:17:34.221256358Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:18:04.230826621Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:18:04.236711158Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:18:31.088505977Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:18:31.096553898Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:18:34.245234623Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:18:34.252337995Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:19:04.265727913Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:19:04.274553011Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:19:31.10869476Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:19:31.11734463Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:19:34.282894992Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:19:34.288340805Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:20:04.300211049Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:20:04.306104496Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:20:31.130367967Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:20:31.137793756Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:20:34.313987849Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:20:34.319649869Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:21:04.331673799Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:21:04.338497995Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:21:31.148525059Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:21:31.155777627Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:21:34.347895469Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:21:34.354858439Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:22:04.363652142Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:22:04.368887159Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:22:31.174981434Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:22:31.18177773Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:22:34.379716459Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:22:34.386079758Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:23:04.398875077Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:23:04.405910114Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:23:31.192745634Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:23:31.200185975Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:23:34.416667392Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:23:34.422440492Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:24:04.433100644Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:24:04.440642697Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:24:31.217935962Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:24:31.224226561Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:24:34.448273348Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:24:34.45387921Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:25:04.466501801Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:25:04.472968581Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:25:31.235685282Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:25:31.243224456Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:25:34.482482251Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:25:34.487972665Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:26:04.498740617Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:26:04.505714442Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:26:31.259850125Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:26:31.267621162Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:26:34.516410951Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:26:34.52186871Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:27:04.532473534Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:27:04.538290488Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:27:31.280078991Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:27:31.290125849Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:27:34.546973697Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:27:34.552593285Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:28:04.564394175Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:28:04.570166666Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:28:31.312154569Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:28:31.320494348Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:28:34.580246473Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:28:34.585749265Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:29:04.598418897Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:29:04.605354988Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:29:31.331305241Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:29:31.337769038Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:29:34.614388775Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:29:34.619892069Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:30:04.630430209Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:30:04.636225729Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:30:31.346312635Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:30:31.352768398Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:30:34.645613305Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:30:34.651139936Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:31:04.662757141Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:31:04.669846667Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:31:31.364227729Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:31:31.371463417Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:31:34.678165042Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:31:34.683878369Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:32:04.695863823Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:32:04.702729724Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:32:31.389647827Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:32:31.399395772Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:32:34.712484398Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:32:34.717919294Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:33:04.727732896Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:33:04.73453039Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:33:31.412116585Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:33:31.41986084Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:33:34.745508174Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:33:34.752821866Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:34:04.765181692Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:34:04.770469974Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:34:31.440234553Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:34:31.448454848Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:34:34.779937766Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:34:34.78558759Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:35:04.797519282Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:35:04.803082599Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:35:31.459287143Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:35:31.467950217Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:35:34.81286783Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:35:34.819199674Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:36:04.830738539Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:36:04.836981656Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:36:31.492331398Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:36:31.501198768Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:36:34.844999881Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:36:34.850856584Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:37:04.861119127Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:37:04.867532417Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:37:31.516639385Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:37:31.52435354Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:37:34.876571049Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:37:34.882000245Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:38:04.893296975Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:38:04.899174218Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:38:31.545410309Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:38:31.553781849Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:38:34.90513113Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:38:34.91026044Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:39:04.921278921Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:39:04.927269281Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:39:31.564848357Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:39:31.572138019Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:39:34.936193883Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:39:34.941426183Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:40:04.95353525Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:40:04.959507131Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:40:31.593795895Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:40:31.603325531Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:40:34.967289852Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:40:34.972224312Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:41:04.983141456Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:41:04.989240498Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:41:31.618246164Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:41:31.625239513Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:41:34.995400277Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:41:35.002507285Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:42:05.017570451Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:42:05.026370743Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:42:31.636647457Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:42:31.644894004Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:42:35.040127734Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:42:35.047532257Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:43:05.057683187Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:43:05.063275476Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:43:31.659110937Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:43:31.667497359Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:43:35.073880306Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:43:35.080351683Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:44:05.092137532Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:44:05.100510903Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:44:31.687533807Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:44:31.696568669Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:44:35.110584884Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:44:35.11766436Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:45:05.130849808Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:45:05.137204054Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:45:31.708718219Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:45:31.718210286Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:45:35.143278536Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:45:35.148990552Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:46:05.15960693Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:46:05.165578714Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:46:31.740574548Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:46:31.750145643Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:46:35.175844033Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:46:35.182996892Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:47:05.193535595Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:47:05.199527596Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:47:31.764376976Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:47:31.772081843Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:47:35.209837351Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:47:35.216847998Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:48:05.225631406Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:48:05.231675156Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:48:31.78527062Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:48:31.792255598Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:48:35.240775241Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:48:35.247835289Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:49:05.257115477Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:49:05.262931364Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:49:31.803760941Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:49:31.811544866Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:49:35.269958138Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:49:35.280106634Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:50:05.290959218Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:50:05.297740113Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:50:31.824543614Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:50:31.832597591Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:50:35.306532604Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:50:35.312370684Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:51:05.323282194Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:51:05.329905949Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:51:31.844151365Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:51:31.851383403Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:51:35.336513823Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:51:35.342313912Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:52:05.353624937Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:52:05.360916135Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:52:31.868532471Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:52:31.875445656Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:52:35.367939527Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:52:35.374067549Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:53:05.383318982Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:53:05.389654784Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:53:31.887317877Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:53:31.895943247Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:53:35.397636983Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:53:35.403847412Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:54:05.414898781Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:54:05.421299394Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:54:31.916082545Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:54:31.923725877Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:54:35.428847945Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:54:35.43498861Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:55:05.451781656Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:55:05.461603564Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:55:31.934336006Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:55:31.941774324Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:55:35.469848267Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:55:35.475378455Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:56:05.489087812Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:56:05.497669483Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:56:31.96148904Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:56:31.970428974Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:56:35.50657127Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:56:35.513995832Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:57:05.523455555Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:57:05.528833535Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:57:31.981096996Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:57:31.988806165Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:57:35.535500956Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:57:35.540727159Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:58:05.550285397Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:58:05.556691055Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:58:32.000911363Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:58:32.009391641Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:58:35.563657559Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:58:35.569686974Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:59:05.579080689Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:59:05.584697995Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T01:59:32.022642324Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:59:32.029777816Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T01:59:35.592857462Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T01:59:35.598807421Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:00:05.613767844Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:00:05.621999193Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:00:32.050355003Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:00:32.057949651Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:00:35.63144169Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:00:35.639215125Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:01:05.649333936Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:01:05.655850056Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:01:32.075369707Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:01:32.089479527Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:01:35.662874348Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:01:35.668752254Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:02:05.677063156Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:02:05.682992461Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:02:32.111608204Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:02:32.11880087Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:02:35.690416001Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:02:35.698654364Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:03:05.708279133Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:03:05.71365651Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:03:32.140469879Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:03:32.147370134Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:03:35.719914379Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:03:35.726136364Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:04:05.734698708Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:04:05.740436995Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:04:32.165162429Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:04:32.172597345Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:04:35.748338365Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:04:35.753566711Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:05:05.765725646Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:05:05.77336366Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:05:32.183685688Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:05:32.192023546Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:05:35.782642564Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:05:35.791102923Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:06:05.801287222Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:06:05.807935061Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:06:32.210204348Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:06:32.217216217Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:06:35.817101684Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:06:35.825293961Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:07:05.837640252Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:07:05.846386395Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:07:32.228554903Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:07:32.236073291Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:07:35.853935638Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:07:35.861637598Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:08:05.870286911Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:08:05.876493511Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:08:32.251862274Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:08:32.262505078Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:08:35.882810619Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:08:35.888233354Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:09:05.896854318Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:09:05.902841454Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:09:32.273531029Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:09:32.280524859Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:09:35.910723459Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:09:35.917303576Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:10:05.927639679Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:10:05.934373843Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:10:32.296841792Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:10:32.304127709Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:10:35.94228069Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:10:35.948373728Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:11:05.958546954Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:11:05.96428011Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:11:32.313952074Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:11:32.320626413Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:11:35.972269993Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:11:35.979653087Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:12:05.988065548Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:12:05.993427651Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:12:32.339141428Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:12:32.346156455Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:12:36.000972564Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:12:36.007769947Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:13:06.02030612Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:13:06.026737809Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:13:32.357192404Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:13:32.364576735Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:13:36.035593852Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:13:36.042827866Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:14:06.051029527Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:14:06.05660765Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:14:32.383892944Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:14:32.392835059Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:14:36.063837069Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:14:36.070292413Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:15:06.079419652Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:15:06.085417941Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:15:32.404694507Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:15:32.412495268Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:15:36.093466856Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:15:36.100212316Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:16:06.109431256Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:16:06.115797048Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:16:32.430322413Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:16:32.437672343Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:16:36.122710007Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:16:36.128073884Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:17:06.136559999Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:17:06.142355425Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:17:32.450236719Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:17:32.458318648Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:17:36.152195429Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:17:36.160774526Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:18:06.171504059Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:18:06.178754801Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:18:32.47263143Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:18:32.480429264Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:18:36.186157612Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:18:36.192007865Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:19:06.201147034Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:19:06.206753272Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:19:32.495171071Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:19:32.504352593Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:19:36.217244627Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:19:36.226179486Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:20:06.235897668Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:20:06.242354061Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:20:32.525014102Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:20:32.532438499Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:20:36.250022721Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:20:36.257217767Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:21:06.267261984Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:21:06.273970594Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:21:32.545159196Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:21:32.551971548Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:21:36.281763517Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:21:36.288598836Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:22:06.298788007Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:22:06.304554849Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:22:32.573152393Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:22:32.580519955Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:22:36.312848886Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:22:36.318915869Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:23:06.329940666Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:23:06.336354951Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:23:32.593758995Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:23:32.600774331Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:23:36.346070227Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:23:36.35543693Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:24:06.36887155Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:24:06.376190454Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:24:32.618000954Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:24:32.6243468Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:24:36.387780801Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:24:36.395495002Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:25:06.404455198Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:25:06.411024606Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:25:32.638236583Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:25:32.647075857Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:25:36.418884974Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:25:36.425482933Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:26:06.435331431Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:26:06.441290119Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:26:32.667988016Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:26:32.674599186Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:26:36.449727374Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:26:36.456084191Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:27:06.467308701Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:27:06.475226134Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:27:32.6852343Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:27:32.692478986Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:27:36.484903453Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:27:36.495172875Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:28:06.504949507Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:28:06.510765918Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:28:32.712931259Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:28:32.719802586Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:28:36.519485186Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:28:36.526289186Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:29:06.535802628Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:29:06.541493172Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:29:32.732201976Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:29:32.739483779Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:29:36.548020231Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:29:36.554311304Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:30:06.564168662Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:30:06.569661047Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:30:32.750686274Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:30:32.75813545Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:30:36.577362041Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:30:36.582978424Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:31:06.592561246Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:31:06.59940371Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:31:32.769875421Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:31:32.777775966Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:31:36.607504496Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:31:36.613287598Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:32:06.625067245Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:32:06.633207165Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:32:32.800916003Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:32:32.808431569Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:32:36.644590602Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:32:36.656347759Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:33:06.668571408Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:33:06.675023124Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:33:32.819261821Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:33:32.826946271Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:33:36.682538609Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:33:36.689091155Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:34:06.699349128Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:34:06.705738683Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:34:32.841247483Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:34:32.848989591Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:34:36.711698361Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:34:36.71706404Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:35:06.726540657Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:35:06.732502929Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:35:32.860290859Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:35:32.867417398Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:35:36.741097131Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:35:36.748813738Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:36:06.758845697Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:36:06.764282618Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:36:32.889251769Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:36:32.898649133Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:36:36.773512398Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:36:36.781120491Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:37:06.791961396Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:37:06.798121812Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:37:32.909637339Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:37:32.918156442Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:37:36.80668585Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:37:36.817079687Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:38:06.829750445Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:38:06.837555017Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:38:32.939169171Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:38:32.946389735Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:38:36.844719996Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:38:36.849532241Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:39:06.857742056Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:39:06.863423313Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:39:32.958529222Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:39:32.965798987Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:39:36.873412954Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:39:36.881255372Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:40:06.891747272Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:40:06.901415176Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:40:32.987513575Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:40:32.995415718Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:40:36.909328724Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:40:36.914770773Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:41:06.923643416Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:41:06.930636909Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:41:33.00838159Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:41:33.016494563Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:41:36.937911069Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:41:36.943689901Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:42:06.952948712Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:42:06.95862656Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:42:33.037200254Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:42:33.043807347Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:42:36.966916222Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:42:36.97459969Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:43:06.98509121Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:43:06.992027721Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:43:33.058756928Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:43:33.069091964Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:43:36.999496049Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:43:37.005858219Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:44:07.017674306Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:44:07.023479925Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:44:33.090754814Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:44:33.098414913Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:44:37.032735962Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:44:37.039148825Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:45:07.047658896Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:45:07.052685827Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:45:33.1111789Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:45:33.120132855Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:45:37.062959687Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:45:37.074200917Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:46:07.086971682Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:46:07.094591619Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:46:33.143953645Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:46:33.151867589Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:46:37.112317054Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:46:37.118253186Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:47:07.127675037Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:47:07.135006968Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:47:33.163448265Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:47:33.171114396Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:47:37.143833136Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:47:37.155166675Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:48:07.165886438Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:48:07.172173911Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:48:33.185959872Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:48:33.194320792Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:48:37.18059832Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:48:37.191855568Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:49:07.205916363Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:49:07.214876566Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:49:33.207519161Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:49:33.217596156Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:49:37.221848051Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:49:37.227300124Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:50:07.238503815Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:50:07.244710216Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:50:33.239062839Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:50:33.246081503Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:50:37.253872725Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:50:37.259700859Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:51:07.273658598Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:51:07.282284729Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:51:33.258178114Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:51:33.26556858Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:51:37.290369963Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:51:37.295968037Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:52:07.306596102Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:52:07.312501832Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:52:33.289338643Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:52:33.29832878Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:52:37.319106338Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:52:37.325747305Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:53:07.336975979Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:53:07.344218883Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:53:33.31272074Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:53:33.321069591Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:53:37.354698579Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:53:37.362824131Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:54:07.372283673Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:54:07.378613502Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:54:33.345723353Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:54:33.357451881Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:54:37.387221098Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:54:37.392278488Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:55:07.401062078Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:55:07.40640195Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:55:33.368659891Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:55:33.375915353Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:55:37.414232685Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:55:37.419632237Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:56:07.429534422Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:56:07.435455522Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:56:33.390471993Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:56:33.399394889Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:56:37.444280937Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:56:37.450594174Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:57:07.458654712Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:57:07.465253682Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:57:33.409574997Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:57:33.417526787Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:57:37.472848329Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:57:37.480548571Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:58:07.490890337Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:58:07.497937818Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:58:33.438590545Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:58:33.445769463Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:58:37.508374631Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:58:37.515035852Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:59:07.52820916Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:59:07.536494497Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T02:59:33.458885368Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:59:33.468547792Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T02:59:37.544116826Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T02:59:37.550522465Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:00:07.562331703Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:00:07.56944132Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:00:33.489832564Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:00:33.497122921Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:00:37.577317499Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:00:37.584747527Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:01:07.598677476Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:01:07.607983454Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:01:33.509615437Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:01:33.517643251Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:01:37.617565955Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:01:37.626558748Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:02:07.640122965Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:02:07.66381111Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:02:33.53977734Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:02:33.548639689Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:02:37.672604287Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:02:37.679343951Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:03:07.696378537Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:03:07.705062054Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:03:33.560639989Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:03:33.569135224Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:03:37.712008049Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:03:37.717900186Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:04:07.726888916Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:04:07.737343519Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:04:33.592525658Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:04:33.601624449Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:04:37.745785409Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:04:37.75172426Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:05:07.762337208Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:05:07.767906768Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:05:33.613206874Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:05:33.622994234Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:05:37.776241202Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:05:37.782335574Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:06:07.793462345Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:06:07.798770387Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:06:33.642407784Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:06:33.649674921Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:06:37.806976173Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:06:37.813510054Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:07:07.824892994Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:07:07.831551645Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:07:33.660954345Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:07:33.668359869Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:07:37.840854461Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:07:37.84679779Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:08:07.857627787Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:08:07.864267895Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:08:33.690494447Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:08:33.697627092Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:08:37.873934606Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:08:37.879648909Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:09:07.89036329Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:09:07.896662299Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:09:33.708465658Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:09:33.716520809Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:09:37.904656106Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:09:37.910134674Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:10:07.920617129Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:10:07.926005009Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:10:33.734908259Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:10:33.742128287Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:10:37.934379965Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:10:37.940467953Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:11:07.95329985Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:11:07.960580356Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:11:33.752673199Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:11:33.761573213Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:11:37.968677954Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:11:37.974575039Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:12:07.986374272Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:12:07.992060247Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:12:33.77944423Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:12:33.786918205Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:12:38.000826397Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:12:38.008415752Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:13:08.022999376Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:13:08.029149933Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:13:33.798130901Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:13:33.805976307Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:13:38.03609091Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:13:38.042297168Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:14:08.052875522Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:14:08.058344033Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:14:33.828090527Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:14:33.834896625Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:14:38.089016797Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:14:38.122939164Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:15:08.132453049Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:15:08.137545877Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:15:33.845733718Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:15:33.853182106Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:15:38.146983965Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:15:38.152932311Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:16:08.162526257Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:16:08.168295986Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:16:33.872153347Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:16:33.879564674Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:16:38.176874952Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:16:38.184318231Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:17:08.19662974Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:17:08.203300866Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:17:33.888970697Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:17:33.897322425Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:17:38.211743221Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:17:38.218121311Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:18:08.230219258Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:18:08.235953142Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:18:33.90801267Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:18:33.915158184Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:18:38.245731362Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:18:38.251928917Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:19:08.263024654Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:19:08.269445087Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:19:33.931216846Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:19:33.940231126Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:19:38.278901067Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:19:38.287203481Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:20:08.29785105Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:20:08.303508057Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:20:33.963426315Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:20:33.971275736Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:20:38.311566365Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:20:38.316788795Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:21:08.327758485Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:21:08.333616297Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:21:33.981774556Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:21:33.988783018Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:21:38.343156003Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:21:38.348529312Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:22:08.359364942Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:22:08.36536068Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:22:34.011914719Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:22:34.020117272Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:22:38.376005946Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:22:38.381401735Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:23:08.392833096Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:23:08.398817532Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:23:34.030691155Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:23:34.037494795Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:23:38.408287507Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:23:38.413849556Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:24:08.42228648Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:24:08.427584833Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:24:34.058635859Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:24:34.066877681Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:24:38.436551852Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:24:38.442201Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:25:08.454554563Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:25:08.460997785Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:25:34.080598864Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:25:34.08901913Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:25:38.470993316Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:25:38.477211423Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:26:08.489308331Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:26:08.495598503Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:26:34.113154794Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:26:34.122368071Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:26:38.506016788Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:26:38.514655467Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:27:08.523914982Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:27:08.530429715Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:27:34.132471015Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:27:34.13988669Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:27:38.540497073Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:27:38.546503466Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:28:08.555629288Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:28:08.561379408Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:28:34.159553275Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:28:34.167468393Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:28:38.571886916Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:28:38.578250998Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:29:08.588252915Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:29:08.594925387Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:29:34.179505479Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:29:34.186386783Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:29:38.604845562Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:29:38.611498606Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:30:08.622486743Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:30:08.628129156Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:30:34.207731567Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:30:34.214950284Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:30:38.636899969Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:30:38.642521025Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:31:08.650870724Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:31:08.657184836Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:31:34.225464357Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:31:34.233425312Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:31:38.668645633Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:31:38.675915231Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:32:08.68581879Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:32:08.691932989Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:32:34.25449895Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:32:34.262126799Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:32:38.699825252Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:32:38.706677714Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:33:08.715839271Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:33:08.721497459Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:33:34.274248082Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:33:34.281839094Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:33:38.733120476Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:33:38.739384928Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:34:08.752712489Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:34:08.759783856Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:34:34.30306275Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:34:34.31102799Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:34:38.768637457Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:34:38.775021311Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:35:08.787195566Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:35:08.794962628Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:35:34.323295847Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:35:34.330171707Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:35:38.804138049Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:35:38.810173957Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:36:08.819823506Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:36:08.825284328Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:36:34.353239312Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:36:34.360316596Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:36:38.832211089Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:36:38.838106619Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:37:08.849975829Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:37:08.855807219Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:37:34.372430706Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:37:34.380180856Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:37:38.865007595Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:37:38.870274146Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:38:08.882262598Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:38:08.888212971Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:38:34.403522067Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:38:34.411494122Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:38:38.899479909Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:38:38.908686108Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:39:08.921744795Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:39:08.92985691Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:39:34.421862852Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:39:34.429129777Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:39:38.938250579Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:39:38.943943151Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:40:08.953240283Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:40:08.959007926Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:40:34.452354979Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:40:34.460207006Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:40:38.970684832Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:40:38.976291644Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:41:08.986702803Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:41:08.99375987Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:41:34.470928873Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:41:34.479279044Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:41:39.00425971Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:41:39.009920848Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:42:09.022852293Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:42:09.029879931Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:42:34.500756425Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:42:34.509878986Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:42:39.041838506Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:42:39.050098694Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:43:09.072075055Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:43:09.07945825Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:43:34.524817032Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:43:34.534568931Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:43:39.086335608Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:43:39.091443363Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:44:09.104653703Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:44:09.110850403Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:44:34.561288704Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:44:34.572380523Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:44:39.119056839Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:44:39.124020912Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:45:09.140520856Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:45:09.14930626Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:45:34.587125642Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:45:34.595072269Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:45:39.15897511Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:45:39.16453363Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:46:09.177207146Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:46:09.184434103Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:46:34.617462141Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:46:34.624788316Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:46:39.194206535Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:46:39.201474937Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:47:09.214457817Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:47:09.221240083Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:47:34.637541993Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:47:34.646735534Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:47:39.23250716Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:47:39.238740776Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:48:09.249768787Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:48:09.25676413Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:48:34.664934079Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:48:34.672852992Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:48:39.262965134Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:48:39.268432186Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:49:09.281633825Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:49:09.288551405Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:49:34.684099146Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:49:34.6938673Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:49:39.295306667Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:49:39.301549254Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:50:09.310962518Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:50:09.316833093Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:50:34.714948961Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:50:34.722014987Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:50:39.327120175Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:50:39.335186963Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:51:09.346684431Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:51:09.353068648Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:51:34.733213472Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:51:34.740054532Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:51:39.36110011Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:51:39.367294744Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:52:09.379393716Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:52:09.385299807Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:52:34.75954494Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:52:34.768364517Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:52:39.394345467Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:52:39.400192572Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:53:09.415668669Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:53:09.42192986Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:53:34.780923852Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:53:34.788626195Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:53:39.436860268Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:53:39.445963156Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:54:09.458717262Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:54:09.464918437Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:54:34.810904592Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:54:34.818836876Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:54:39.47771649Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:54:39.485360741Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:55:09.498806444Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:55:09.505442406Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:55:34.831407772Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:55:34.840682573Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:55:39.51468461Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:55:39.521112332Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:56:09.531920866Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:56:09.537782921Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:56:34.862868645Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:56:34.870285527Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:56:39.544143399Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:56:39.548912899Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:57:09.563599281Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:57:09.569863713Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:57:34.881410121Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:57:34.888760266Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:57:39.581129725Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:57:39.588540653Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:58:09.597959511Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:58:09.606869972Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:58:34.902533068Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:58:34.910416698Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:58:39.619561616Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:58:39.62540278Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:59:09.638220437Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:59:09.644702105Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T03:59:34.922130427Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:59:34.929973822Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T03:59:39.652508462Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T03:59:39.659293782Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:00:09.675970286Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:00:09.68413573Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:00:34.955385346Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:00:34.964230645Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:00:39.695466473Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:00:39.701797791Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:01:09.710487008Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:01:09.716753947Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:01:34.976326805Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:01:34.984082466Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:01:39.727801686Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:01:39.734581912Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:02:09.746840044Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:02:09.752518922Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:02:35.008823134Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:02:35.019487822Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:02:39.759853287Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:02:39.765252793Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:03:09.779105572Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:03:09.784806466Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:03:35.032705577Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:03:35.041213724Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:03:39.793738607Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:03:39.799341401Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:04:09.812315726Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:04:09.81929783Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:04:35.057714838Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:04:35.064791034Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:04:39.829991547Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:04:39.835683832Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:05:09.847793139Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:05:09.854745812Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:05:35.076469899Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:05:35.083744156Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:05:39.868087867Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:05:39.87651723Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:06:09.888894069Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:06:09.894007543Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:06:35.106726843Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:06:35.114130184Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:06:39.90220452Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:06:39.909933202Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:07:09.918801979Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:07:09.924540862Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:07:35.124124379Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:07:35.130725091Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:07:39.933958369Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:07:39.938706775Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:08:09.947942933Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:08:09.953651495Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:08:35.148109848Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:08:35.156521259Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:08:39.963478326Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:08:39.969196598Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:09:09.98022112Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:09:09.985447278Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:09:35.167148783Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:09:35.174457656Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:09:39.992403803Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:09:39.998301025Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:10:10.011341008Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:10:10.018228592Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:10:35.194774648Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:10:35.201561324Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:10:40.024458484Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:10:40.030877135Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:11:10.04119265Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:11:10.047104497Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:11:35.212823334Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:11:35.221554813Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:11:40.05850795Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:11:40.064276197Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:12:10.073861792Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:12:10.079922644Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:12:35.233653386Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:12:35.242482288Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:12:40.093880752Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:12:40.108275496Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:13:10.121030981Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:13:10.128445068Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:13:35.253203537Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:13:35.260261625Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:13:40.143069324Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:13:40.150663332Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:14:10.162942448Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:14:10.171222532Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:14:35.269323735Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:14:35.276463224Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:14:40.181566919Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:14:40.187582966Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:15:10.199395095Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:15:10.205831706Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:15:35.285813041Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:15:35.293290182Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:15:40.212172962Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:15:40.217434885Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:16:10.227909416Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:16:10.233440658Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:16:35.302421332Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:16:35.309947228Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:16:40.244012897Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:16:40.249727038Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:17:10.26342493Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:17:10.270160596Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:17:35.323325592Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:17:35.330540745Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:17:40.283578998Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:17:40.290742374Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:18:10.305549703Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:18:10.31261299Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:18:35.341217778Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:18:35.348174635Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:18:40.323272031Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:18:40.328867311Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:19:10.341474995Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:19:10.347487631Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:19:35.359335368Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:19:35.366856533Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:19:40.36017076Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:19:40.365820715Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:20:10.377526428Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:20:10.382801466Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:20:35.376458631Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:20:35.384748873Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:20:40.38976608Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:20:40.396458586Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:21:10.409495919Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:21:10.416349414Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:21:35.399875198Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:21:35.408955213Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:21:40.423129947Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:21:40.428376904Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:22:10.443592017Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:22:10.451128255Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:22:35.417612785Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:22:35.424891698Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:22:40.460691564Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:22:40.468425924Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:23:10.477864378Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:23:10.484210604Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:23:35.436726107Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:23:35.444949753Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:23:40.492134607Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:23:40.499736143Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:24:10.517523793Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:24:10.524073352Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:24:35.454595952Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:24:35.461154932Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:24:40.532758505Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:24:40.5397561Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:25:10.55247049Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:25:10.558620916Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:25:35.471230436Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:25:35.478904704Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:25:40.568162999Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:25:40.574657849Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:26:10.591523378Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:26:10.597421159Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:26:35.488804739Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:26:35.49624712Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:26:40.604984761Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:26:40.613388079Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:27:10.625980575Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:27:10.632259602Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:27:35.50884673Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:27:35.515989679Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:27:40.640926241Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:27:40.647206876Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:28:10.660114558Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:28:10.666697555Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:28:35.530246888Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:28:35.540250033Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:28:40.67751108Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:28:40.684122884Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:29:10.697619866Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:29:10.703858807Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:29:35.552722817Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:29:35.55993737Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:29:40.711664773Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:29:40.717429252Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:30:10.727383887Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:30:10.734779692Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:30:35.569140638Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:30:35.576016113Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:30:40.745335316Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:30:40.752693249Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:31:10.764507649Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:31:10.771547932Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:31:35.590277858Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:31:35.601939619Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:31:40.782898504Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:31:40.789622322Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:32:10.800902053Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:32:10.805937344Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:32:35.612396072Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:32:35.621817175Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:32:40.816509506Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:32:40.822979852Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:33:10.833271873Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:33:10.839477769Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:33:35.636020129Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:33:35.64253505Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:33:40.847322253Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:33:40.85465224Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:34:10.865726448Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:34:10.871122421Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:34:35.652565772Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:34:35.659949652Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:34:40.882317628Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:34:40.888528571Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:35:10.900724492Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:35:10.906700813Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:35:35.671593856Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:35:35.680692836Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:35:40.913004047Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:35:40.918139374Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:36:10.935023676Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:36:10.942625868Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:36:35.691157017Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:36:35.701310079Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:36:40.950391833Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:36:40.956516183Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:37:10.968137296Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:37:10.975848507Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:37:35.71184569Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:37:35.72021107Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:37:40.986431273Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:37:40.992401734Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:38:11.005463948Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:38:11.010884507Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:38:35.730593108Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:38:35.738256224Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:38:41.023365765Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:38:41.029604652Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:39:11.041272936Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:39:11.046418558Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:39:35.752486824Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:39:35.761314168Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:39:41.053732221Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:39:41.059960439Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:40:11.06891367Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:40:11.075755572Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:40:35.772360662Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:40:35.779581557Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:40:41.083680247Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:40:41.089415838Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:41:11.10412596Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:41:11.10982879Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:41:35.795108232Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:41:35.803539755Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:41:41.122174183Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:41:41.128121124Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:42:11.138394096Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:42:11.144338215Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:42:35.812294553Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:42:35.819590053Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:42:41.15140629Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:42:41.157633193Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:43:11.166764435Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:43:11.172651116Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:43:35.832334142Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:43:35.839560227Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:43:41.180163677Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:43:41.186066179Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:44:11.200841343Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:44:11.207112116Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:44:35.848329454Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:44:35.856326446Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:44:41.218363502Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:44:41.224535624Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:45:11.241531865Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:45:11.249827469Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:45:35.869955588Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:45:35.877381871Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:45:41.260498164Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:45:41.266162447Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:46:11.279451049Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:46:11.285707066Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:46:35.886754043Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:46:35.893569536Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:46:41.297661105Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:46:41.303816734Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:47:11.312600217Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:47:11.318207174Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:47:35.9045682Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:47:35.912415477Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:47:41.33137573Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:47:41.337996959Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:48:11.346949021Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:48:11.353979953Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:48:35.923225535Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:48:35.933609169Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:48:41.361255164Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:48:41.367494809Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:49:11.38059992Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:49:11.386579738Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:49:35.947725618Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:49:35.954909413Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:49:41.394365397Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:49:41.400628812Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:50:11.418588875Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:50:11.426124013Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:50:35.963579641Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:50:35.971557867Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:50:41.436206805Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:50:41.444821063Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:51:11.457616353Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:51:11.465088118Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:51:35.988768808Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:51:35.999906265Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:51:41.476697993Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:51:41.484119305Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:52:11.503275622Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:52:11.510111745Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:52:36.009957214Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:52:36.017176537Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:52:41.523093529Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:52:41.530980098Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:53:11.540583834Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:53:11.545704066Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:53:36.027866135Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:53:36.035989137Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:53:41.55359728Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:53:41.559649337Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:54:11.575297019Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:54:11.582023837Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:54:36.048485176Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:54:36.058988336Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:54:41.58942766Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:54:41.598130112Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:55:11.615320383Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:55:11.623090014Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:55:36.071669958Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:55:36.078890126Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:55:41.637597818Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:55:41.646933738Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:56:11.655467819Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:56:11.661959268Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:56:36.087135627Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:56:36.0943134Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:56:41.671759908Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:56:41.679555429Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:57:11.692818178Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:57:11.698259856Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:57:36.106983241Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:57:36.115921589Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:57:41.705648868Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:57:41.711710668Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:58:11.722332395Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:58:11.729779197Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:58:36.126341372Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:58:36.134376721Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:58:41.741507641Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:58:41.747220586Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:59:11.759221942Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:59:11.764712472Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T04:59:36.14683965Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:59:36.153739346Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T04:59:41.774845075Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T04:59:41.781969618Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:00:11.790368959Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:00:11.796438201Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:00:36.163502906Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:00:36.170888978Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:00:41.804704332Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:00:41.811014662Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:01:11.824801244Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:01:11.830822096Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:01:36.181692321Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:01:36.189093305Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:01:41.837676771Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:01:41.843005303Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:02:11.851360669Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:02:11.857239292Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:02:36.19857259Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:02:36.206004892Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:02:41.868159325Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:02:41.874712823Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:03:11.884987394Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:03:11.891345909Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:03:36.220098561Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:03:36.231316127Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:03:41.902197976Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:03:41.908698872Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:04:11.920532168Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:04:11.927915512Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:04:36.241897357Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:04:36.249722371Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:04:41.934542344Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:04:41.939810102Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:05:11.952344366Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:05:11.957965936Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:05:36.265911515Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:05:36.275543214Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:05:41.966448804Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:05:41.972213096Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:06:11.981385656Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:06:11.987301508Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:06:36.286139996Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:06:36.292486641Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:06:41.998911945Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:06:42.004780683Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:07:12.017726731Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:07:12.023360644Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:07:36.303403424Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:07:36.311162745Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:07:42.031695321Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:07:42.037558399Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:08:12.046971351Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:08:12.054164421Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:08:36.32412703Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:08:36.334797734Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:08:42.066136077Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:08:42.071659493Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:09:12.082282849Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:09:12.088389013Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:09:36.35374494Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:09:36.363451568Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:09:42.102907581Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:09:42.110197553Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:10:12.123972133Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:10:12.129487445Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:10:36.372483689Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:10:36.379217397Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:10:42.140584451Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:10:42.147102479Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:11:12.157624045Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:11:12.16423699Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:11:36.392849392Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:11:36.40274938Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:11:42.175194786Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:11:42.180991003Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:12:12.194253017Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:12:12.199790225Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:12:36.411838514Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:12:36.419543669Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:12:42.208393602Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:12:42.215721255Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:13:12.22498069Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:13:12.231053313Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:13:36.436470717Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:13:36.445546295Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:13:42.239519702Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:13:42.244712439Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:14:12.257246375Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:14:12.262841145Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:14:36.459941611Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:14:36.473328169Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:14:42.275781221Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:14:42.28144409Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:15:12.290278465Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:15:12.295523934Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:15:36.48720689Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:15:36.494889666Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:15:42.30736748Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:15:42.3122971Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:16:12.326438094Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:16:12.332461526Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:16:36.503029723Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:16:36.510760691Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:16:42.343967682Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:16:42.348955499Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:17:12.363962433Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:17:12.371217244Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:17:36.526451478Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:17:36.538360593Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:17:42.379189445Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:17:42.384946706Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:18:12.398439158Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:18:12.404420527Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:18:36.549284354Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:18:36.556218303Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:18:42.417089593Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:18:42.423811584Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:19:12.437868604Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:19:12.44420785Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:19:36.567628066Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:19:36.575102638Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:19:42.455848308Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:19:42.462197888Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:20:12.478201826Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:20:12.48456835Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:20:36.584950669Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:20:36.591596659Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:20:42.506730149Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:20:42.517988495Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:21:12.531584773Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:21:12.538447057Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:21:36.604282168Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:21:36.613416498Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:21:42.545749809Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:21:42.552131302Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:22:12.566416526Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:22:12.572153429Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:22:36.62316921Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:22:36.629419406Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:22:42.586346219Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:22:42.592986676Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:23:12.605474686Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:23:12.611232932Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:23:36.64056783Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:23:36.647589081Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:23:42.618955714Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:23:42.626985936Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:24:12.635204337Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:24:12.641449266Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:24:36.659128703Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:24:36.667094966Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:24:42.65310661Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:24:42.659063181Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:25:12.668800177Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:25:12.675092472Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:25:36.680361479Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:25:36.688091916Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:25:42.683264141Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:25:42.689449043Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:26:12.705782689Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:26:12.712494307Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:26:36.69935284Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:26:36.706219203Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:26:42.718823467Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:26:42.723810755Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:27:12.733665058Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:27:12.739332342Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:27:36.71695224Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:27:36.724014676Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:27:42.746793006Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:27:42.752398211Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:28:12.766798782Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:28:12.773289065Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:28:36.73709033Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:28:36.7467657Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:28:42.78482739Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:28:42.790862306Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:29:12.803825924Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:29:12.81018154Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:29:36.758597043Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:29:36.765488431Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:29:42.819786936Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:29:42.826306466Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:30:12.834822428Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:30:12.840651954Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:30:36.775664488Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:30:36.782035862Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:30:42.852125431Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:30:42.857214259Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:31:12.870590841Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:31:12.878063284Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:31:36.855205735Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:31:36.865849613Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:31:42.886597123Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:31:42.891806609Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:32:12.906080803Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:32:12.913172661Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:32:36.876319182Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:32:36.883231459Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:32:42.925767991Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:32:42.933518999Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:33:12.945012173Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:33:12.95316313Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:33:36.895502128Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:33:36.90186106Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:33:42.962499944Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:33:42.968945253Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:34:12.982958574Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:34:12.988816667Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:34:36.914503301Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:34:36.92374968Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:34:43.002082488Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:34:43.008759975Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:35:13.02305843Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:35:13.030195417Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:35:36.943833871Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:35:36.955885922Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:35:43.0425287Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:35:43.050810819Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:36:13.066965662Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:36:13.076116905Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:36:36.985880231Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:36:36.995522841Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:36:43.092107008Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:36:43.135956068Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:37:13.150497141Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:37:13.157361418Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:37:37.009544256Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:37:37.016940334Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:37:43.1659546Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:37:43.171915678Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:38:13.183421076Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:38:13.190850788Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:38:37.030984888Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:38:37.037551868Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:38:43.199838898Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:38:43.206338076Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:39:13.218672401Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:39:13.225985722Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:39:37.051789398Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:39:37.060774609Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:39:43.238427783Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:39:43.243906842Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:40:13.257819992Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:40:13.264712913Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:40:37.071217389Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:40:37.078746229Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:40:43.277587452Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:40:43.284061527Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:41:13.296807942Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:41:13.303032612Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:41:37.10989839Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:41:37.119931861Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:41:43.31428135Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:41:43.322351401Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:42:13.332670233Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:42:13.339503197Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:42:37.130376761Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:42:37.136832313Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:42:43.352763573Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:42:43.359673806Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:43:13.370497526Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:43:13.376248918Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:43:37.152185678Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:43:37.15900277Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:43:43.384174477Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:43:43.392239281Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:44:13.410514531Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:44:13.419288619Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:44:37.173288226Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:44:37.182193563Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:44:43.430633314Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:44:43.436351959Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:45:13.456012803Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:45:13.461609264Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:45:37.198937568Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:45:37.205690211Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:45:43.476715632Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:45:43.48483065Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:46:13.50054533Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:46:13.507117482Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:46:37.218205133Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:46:37.225687781Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:46:43.517914868Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:46:43.525020997Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:47:13.537454403Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:47:13.544879774Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:47:37.243170366Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:47:37.251679307Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:47:43.557374459Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:47:43.563628538Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:48:13.579943584Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:48:13.587451983Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:48:37.263527549Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:48:37.272590319Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:48:43.595944592Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:48:43.602970645Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:49:13.619175343Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:49:13.628257755Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:49:37.284854669Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:49:37.291853784Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:49:43.640932149Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:49:43.647361756Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:50:13.663533862Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:50:13.673329293Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:50:37.305696164Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:50:37.315841368Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:50:43.686110229Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:50:43.691572273Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:51:13.705753688Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:51:13.712601779Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:51:37.325883845Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:51:37.332193801Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:51:43.720730044Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:51:43.72695306Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:52:13.743676507Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:52:13.749527667Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:52:37.344452056Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:52:37.352875129Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:52:43.758820495Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:52:43.76595503Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:53:13.781433782Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:53:13.789143425Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:53:37.36640392Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:53:37.374897996Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:53:43.799265569Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:53:43.805788593Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:54:13.821322281Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:54:13.827442587Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:54:37.386665719Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:54:37.393387595Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:54:43.837115151Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:54:43.842879275Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:55:13.855910563Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:55:13.862351881Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:55:37.405017262Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:55:37.4120787Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:55:43.877499693Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:55:43.884952224Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:56:13.900971915Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:56:13.9076852Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:56:37.423090241Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:56:37.429422398Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:56:43.921164237Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:56:43.928408932Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:57:13.942305444Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:57:13.948633469Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:57:37.441529572Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:57:37.448554677Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:57:43.95727144Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:57:43.965419793Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:58:13.979923457Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:58:13.985880044Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:58:37.456849052Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:58:37.462777843Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:58:43.994740863Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:58:44.005010451Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:59:14.018519013Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:59:14.024534588Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T05:59:37.474966811Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:59:37.482618961Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T05:59:44.032400267Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T05:59:44.037772261Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:00:14.051300786Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:00:14.057078758Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:00:37.491595456Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:00:37.497999434Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:00:44.069761145Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:00:44.076457471Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:01:14.090140418Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:01:14.096895505Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:01:37.513571844Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:01:37.519813045Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:01:44.109200333Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:01:44.116627769Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:02:14.131638053Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:02:14.137698333Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:02:37.530637326Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:02:37.537145264Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:02:44.149772859Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:02:44.155249012Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:03:14.169314773Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:03:14.17526291Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:03:37.549757555Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:03:37.556007692Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:03:44.190073505Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:03:44.199360124Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:04:14.208493544Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:04:14.214507629Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:04:37.568401525Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:04:37.575490291Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:04:44.226653542Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:04:44.232372749Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:05:14.246018413Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:05:14.251633979Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:05:37.593258012Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:05:37.600724064Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:05:44.261112741Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:05:44.272606921Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:06:14.285718789Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:06:14.291857901Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:06:37.614552042Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:06:37.621156107Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:06:44.30603931Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:06:44.312794418Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:07:14.328607823Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:07:14.337876246Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:07:37.633111567Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:07:37.64090863Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:07:44.346664935Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:07:44.353769146Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:08:14.363153066Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:08:14.37041446Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:08:37.651127073Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:08:37.65859978Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:08:44.37850197Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:08:44.38502263Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:09:14.400187528Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:09:14.405867377Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:09:37.670368234Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:09:37.677875087Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:09:44.415007025Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:09:44.42107085Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:10:14.434833844Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:10:14.441790797Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:10:37.688771551Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:10:37.696021358Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:10:44.454558179Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:10:44.460083977Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:11:14.476484822Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:11:14.484653455Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:11:37.706876555Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:11:37.714077677Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:11:44.492215275Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:11:44.499110357Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:12:14.50977736Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:12:14.516662778Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:12:37.723819026Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:12:37.731823207Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:12:44.544459587Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:12:44.582219178Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:13:14.599599608Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:13:14.609586786Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:13:37.746244494Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:13:37.75426998Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:13:44.618356964Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:13:44.625430853Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:14:14.639195207Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:14:14.645734974Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:14:37.768427367Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:14:37.778663674Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:14:44.653831055Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:14:44.659711093Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:15:14.668754431Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:15:14.674802134Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:15:37.791608637Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:15:37.799329177Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:15:44.681931343Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:15:44.68772353Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:16:14.700347912Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:16:14.706306255Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:16:37.810850849Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:16:37.819409988Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:16:44.714810768Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:16:44.721022472Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:17:14.734860218Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:17:14.74177807Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:17:37.829546327Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:17:37.837542036Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:17:44.768963237Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:17:44.79396714Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:18:14.809036383Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:18:14.815876323Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:18:37.849601266Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:18:37.858644293Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:18:44.830696866Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:18:44.837156211Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:19:14.851294817Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:19:14.856406615Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:19:37.871069584Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:19:37.879751697Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:19:44.868776191Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:19:44.873988929Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:20:14.887107518Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:20:14.893810458Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:20:37.890257702Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:20:37.897138832Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:20:44.906923074Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:20:44.91259287Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:21:14.929740991Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:21:14.936465202Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:21:37.906448468Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:21:37.913449255Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:21:44.945891484Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:21:44.952084646Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:22:14.964174072Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:22:14.971034456Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:22:37.923018138Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:22:37.931445659Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:22:44.98625746Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:22:44.99452715Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:23:15.007470832Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:23:15.013278224Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:23:37.942332904Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:23:37.948982277Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:23:45.021032683Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:23:45.027835079Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:24:15.042272532Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:24:15.048106164Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:24:37.958367517Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:24:37.967222827Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:24:45.060291066Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:24:45.06602951Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:25:15.083268348Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:25:15.089622572Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:25:37.979583538Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:25:37.987456554Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:25:45.103830419Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:25:45.112061383Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:26:15.126699746Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:26:15.133490621Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:26:37.99758535Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:26:38.00589411Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:26:45.143160683Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:26:45.148458641Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:27:15.163147672Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:27:15.169579989Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:27:38.016556446Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:27:38.024189168Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:27:45.184158112Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:27:45.189883192Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:28:15.202878742Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:28:15.207772063Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:28:38.033103712Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:28:38.040897731Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:28:45.221200459Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:28:45.227124357Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:29:15.242348847Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:29:15.24774861Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:29:38.055485721Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:29:38.064724977Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:29:45.255900058Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:29:45.262634302Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:30:15.277605949Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:30:15.283258777Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:30:38.076899312Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:30:38.085127369Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:30:45.296458205Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:30:45.301453774Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:31:15.319712904Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:31:15.325766133Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:31:38.096471729Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:31:38.104530458Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:31:45.332424327Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:31:45.33750052Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:32:15.349988653Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:32:15.36395057Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:32:38.115208036Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:32:38.123221546Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:32:45.37611336Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:32:45.382736674Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:33:15.422553726Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:33:15.429718216Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:33:38.135093416Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:33:38.142406329Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:33:45.43672684Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:33:45.442098488Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:34:15.456998Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:34:15.462888166Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:34:38.152192526Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:34:38.159124246Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:34:45.496758116Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:34:45.52082652Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:35:15.531610298Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:35:15.538156083Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:35:38.176103171Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:35:38.182962454Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:35:45.546327607Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:35:45.552301467Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:36:15.566803911Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:36:15.573976093Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:36:38.194397363Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:36:38.202410988Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:36:45.588723906Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:36:45.594608752Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:37:15.6056444Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:37:15.613749357Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:37:38.21265846Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:37:38.219844231Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:37:45.621893561Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:37:45.627661092Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:38:15.640020218Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:38:15.645790596Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:38:38.234642167Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:38:38.247298182Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:38:45.65921903Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:38:45.665310878Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:39:15.67780887Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:39:15.684600178Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:39:38.259664453Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:39:38.268036166Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:39:45.696217547Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:39:45.702779628Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:40:15.718060748Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:40:15.724874731Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:40:38.278776306Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:40:38.286623179Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:40:45.736941945Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:40:45.742144025Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:41:15.758615342Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:41:15.766485106Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:41:38.29858078Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:41:38.309371914Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:41:45.772983776Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:41:45.77773838Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:42:15.789121881Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:42:15.79434986Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:42:38.320571624Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:42:38.328028289Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:42:45.808021211Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:42:45.81368587Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:43:15.829441296Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:43:15.835556513Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:43:38.343681976Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:43:38.351312466Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:43:45.84287176Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:43:45.848178639Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:44:15.858346993Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:44:15.864557013Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:44:38.362114761Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:44:38.369723862Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:44:45.874952746Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:44:45.88099357Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:45:15.894967666Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:45:15.901502164Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:45:38.384376484Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:45:38.392977798Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:45:45.909867569Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:45:45.916500835Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:46:15.934200825Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:46:15.941471103Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:46:38.404448758Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:46:38.412011901Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:46:45.954839823Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:46:45.961344348Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:47:15.974421171Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:47:15.979684016Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:47:38.423856969Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:47:38.431162184Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:47:45.988107146Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:47:45.99414017Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:48:16.00816806Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:48:16.01649051Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:48:38.443420367Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:48:38.45184177Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:48:46.030790855Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:48:46.036792349Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:49:16.051807952Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:49:16.058170895Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:49:38.468153763Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:49:38.476332681Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:49:46.069675663Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:49:46.077761475Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:50:16.087862714Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:50:16.093513717Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:50:38.485590737Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:50:38.491886329Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:50:46.110546932Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:50:46.12020265Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:51:16.136084276Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:51:16.142629935Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:51:38.505684587Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:51:38.513581771Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:51:46.154991189Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:51:46.159762438Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:52:16.174617226Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:52:16.180373813Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:52:38.527105137Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:52:38.535507199Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:52:46.191238267Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:52:46.198378083Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:53:16.212569392Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:53:16.218125028Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:53:38.546926233Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:53:38.555219782Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:53:46.230972633Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:53:46.236839247Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:54:16.251740137Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:54:16.259582934Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:54:38.566796806Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:54:38.574304Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:54:46.268529092Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:54:46.275902981Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:55:16.290949872Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:55:16.297683852Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:55:38.595778362Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:55:38.605771276Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:55:46.306103513Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:55:46.312338109Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:56:16.328765729Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:56:16.33523674Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:56:38.61853647Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:56:38.626078233Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:56:46.351260546Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:56:46.357099043Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:57:16.370520353Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:57:16.376579578Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:57:38.635757267Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:57:38.643539306Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:57:46.387094572Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:57:46.394417087Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:58:16.408085685Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:58:16.417893453Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:58:38.656995509Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:58:38.664254153Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:58:46.431625453Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:58:46.437169478Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:59:16.452500317Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:59:16.458865131Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T06:59:38.678739836Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:59:38.685838441Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T06:59:46.467007124Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T06:59:46.472351312Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:00:16.489935616Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:00:16.498881216Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:00:38.694988789Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:00:38.701697374Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:00:46.514072351Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:00:46.521548478Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:01:16.539095468Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:01:16.545695699Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:01:38.713635122Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:01:38.72199865Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:01:46.552510187Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:01:46.557732851Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:02:16.577332261Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:02:16.583751142Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:02:38.735757009Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:02:38.742797887Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:02:46.592488392Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:02:46.597843605Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:03:16.613631165Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:03:16.620062516Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:03:38.75352225Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:03:38.760407561Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:03:46.628778927Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:03:46.636207543Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:04:16.646867129Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:04:16.65339915Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:04:38.770667176Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:04:38.777839763Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:04:46.662846638Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:04:46.669235502Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:05:16.688812262Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:05:16.697914561Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:05:38.79130717Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:05:38.799934571Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:05:46.707853084Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:05:46.713846175Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:06:16.730752909Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:06:16.737681264Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:06:38.81116733Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:06:38.818536608Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:06:46.752663824Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:06:46.760082655Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:07:16.774980499Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:07:16.783616426Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:07:38.829880123Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:07:38.837615936Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:07:46.798993605Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:07:46.805869662Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:08:16.816980005Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:08:16.822575906Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:08:38.848869161Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:08:38.856146418Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:08:46.83530878Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:08:46.840321124Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:09:16.852029862Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:09:16.859181488Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:09:38.865959711Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:09:38.873403389Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:09:46.873543182Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:09:46.880380923Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:10:16.893759844Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:10:16.900087236Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:10:38.88450033Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:10:38.892771994Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:10:46.914867793Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:10:46.920366396Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:11:16.935535192Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:11:16.942267196Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:11:38.904770949Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:11:38.912982721Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:11:46.951374241Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:11:46.957718991Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:12:16.966875377Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:12:16.974129442Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:12:38.926020373Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:12:38.93355114Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:12:46.983855362Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:12:46.989721897Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:13:17.007904397Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:13:17.014473881Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:13:38.945823015Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:13:38.953522388Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:13:47.02324332Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:13:47.02922789Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:14:17.045849791Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:14:17.051821571Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:14:38.962651226Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:14:38.971227422Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:14:47.068081983Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:14:47.077274273Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:15:17.090605612Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:15:17.096522253Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:15:38.986381423Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:15:38.994478363Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:15:47.106268172Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:15:47.111260138Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:16:17.13040306Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:16:17.13828543Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:16:39.007006376Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:16:39.015680292Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:16:47.15225071Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:16:47.160414243Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:17:17.177658841Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:17:17.183099227Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:17:39.027681312Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:17:39.035577237Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:17:47.196975517Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:17:47.204819285Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:18:17.222118436Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:18:17.228432895Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:18:39.046227542Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:18:39.053553222Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:18:47.244277045Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:18:47.251467096Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:19:17.268279518Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:19:17.273887078Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:19:39.063690843Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:19:39.07492769Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:19:47.281228734Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:19:47.287656443Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:20:17.303555387Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:20:17.309012447Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:20:39.085766201Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:20:39.092873345Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:20:47.315341985Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:20:47.32059439Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:21:17.336462127Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:21:17.342176249Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:21:39.104489092Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:21:39.111772668Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:21:47.355871118Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:21:47.361602233Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:22:17.376731844Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:22:17.382823136Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:22:39.124529316Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:22:39.131778504Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:22:47.39086847Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:22:47.396378773Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:23:17.409761946Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:23:17.415021932Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:23:39.142660381Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:23:39.149920927Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:23:47.422636237Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:23:47.428312792Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:24:17.44440596Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:24:17.450922386Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:24:39.159701349Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:24:39.166303079Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:24:47.464685327Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:24:47.471192927Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:25:17.48843373Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:25:17.494888049Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:25:39.176909616Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:25:39.184138863Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:25:47.513503728Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:25:47.520338491Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:26:17.535945969Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:26:17.542590967Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:26:39.195976924Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:26:39.205258693Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:26:47.559500942Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:26:47.574902359Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:27:17.600901633Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:27:17.612641471Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:27:39.216416262Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:27:39.225530269Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:27:47.622146676Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:27:47.631098135Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:28:17.645714585Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:28:17.654160566Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:28:39.238761181Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:28:39.24800524Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:28:47.667882357Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:28:47.67366847Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:29:17.691784717Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:29:17.698784862Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:29:39.260321474Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:29:39.267554791Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:29:47.707916704Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:29:47.713150312Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:30:17.731152812Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:30:17.737258777Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:30:39.279798331Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:30:39.286725457Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:30:47.747950033Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:30:47.753762877Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:31:17.769686201Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:31:17.775383752Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:31:39.298126242Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:31:39.305559741Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:31:47.784386011Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:31:47.791071218Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:32:17.80649769Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:32:17.812297537Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:32:39.3187062Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:32:39.327275293Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:32:47.825599885Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:32:47.833473587Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:33:17.848798294Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:33:17.853949389Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:33:39.340972726Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:33:39.349301585Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:33:47.862008017Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:33:47.867037287Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:34:17.885366478Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:34:17.892849068Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:34:39.359206446Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:34:39.365797063Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:34:47.903757941Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:34:47.910592938Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:35:17.926817168Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:35:17.934172416Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:35:39.376704447Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:35:39.384131956Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:35:47.94467421Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:35:47.951624781Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:36:17.968421565Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:36:17.97452037Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:36:39.392909071Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:36:39.400197199Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:36:47.988101163Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:36:47.993780053Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:37:18.011405294Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:37:18.017776331Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:37:39.41044344Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:37:39.417778985Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:37:48.033104094Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:37:48.040321934Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:38:18.057830366Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:38:18.064038422Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:38:39.431585967Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:38:39.438833626Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:38:48.077656176Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:38:48.084942757Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:39:18.102104564Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:39:18.109780273Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:39:39.450864164Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:39:39.460508977Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:39:48.120684768Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:39:48.12758132Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:40:18.137933242Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:40:18.143840475Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:40:39.472824959Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:40:39.48054455Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:40:48.152544782Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:40:48.159325442Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:41:18.175459372Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:41:18.18132481Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:41:39.493415398Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:41:39.500526071Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:41:48.197182961Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:41:48.203812845Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:42:18.217522037Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:42:18.224558091Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:42:39.512673156Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:42:39.520156511Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:42:48.238113014Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:42:48.243607786Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:43:18.254823349Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:43:18.261944873Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:43:39.538383678Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:43:39.549417254Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:43:48.27929518Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:43:48.28864592Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:44:18.305759078Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:44:18.310781781Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:44:39.564007608Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:44:39.571310403Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:44:48.320738852Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:44:48.326641731Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:45:18.345903642Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:45:18.352161675Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:45:39.586100411Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:45:39.595176271Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:45:48.364276858Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:45:48.371812972Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:46:18.391811105Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:46:18.40284097Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:46:39.607397685Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:46:39.614931888Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:46:48.417246964Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:46:48.424205863Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:47:18.440160414Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:47:18.446627198Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:47:39.625544165Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:47:39.632953775Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:47:48.456497571Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:47:48.462800685Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:48:18.479597643Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:48:18.485279384Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:48:39.643419368Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:48:39.651668991Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:48:48.502406656Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:48:48.508945199Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:49:18.524851357Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:49:18.530149314Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:49:39.662297296Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:49:39.669532591Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:49:48.541478719Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:49:48.551223754Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:50:18.568183954Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:50:18.5744622Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:50:39.678426602Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:50:39.685962626Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:50:48.590290743Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:50:48.598026042Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:51:18.613876107Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:51:18.620402553Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:51:39.697033884Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:51:39.705797265Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:51:48.636761686Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:51:48.644351102Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:52:18.657911878Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:52:18.66313719Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:52:39.714237121Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:52:39.720933317Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:52:48.677487536Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:52:48.682988275Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:53:18.702092163Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:53:18.709185431Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:53:39.735205304Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:53:39.744949444Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:53:48.719209326Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:53:48.72485721Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:54:18.743489614Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:54:18.751693086Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:54:39.759272959Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:54:39.766877535Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:54:48.768064573Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:54:48.777625196Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:55:18.795786132Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:55:18.801185834Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:55:39.779066684Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:55:39.787026417Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:55:48.810692791Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:55:48.815855139Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:56:18.83028608Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:56:18.835285066Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:56:39.801024303Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:56:39.808926851Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:56:48.851225207Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:56:48.857285769Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:57:18.873465419Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:57:18.880509128Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:57:39.819769287Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:57:39.827777855Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:57:48.890909804Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:57:48.896430696Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:58:18.914390347Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:58:18.92061275Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:58:39.840343161Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:58:39.848858289Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:58:48.932034125Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:58:48.938585718Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:59:18.955487867Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:59:18.961258448Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T07:59:39.859135494Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:59:39.866643515Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T07:59:48.973033063Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T07:59:48.979328427Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:00:18.996881123Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:00:19.003510435Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:00:39.876356923Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:00:39.884449973Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:00:49.011890685Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:00:49.021876789Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:01:19.038955076Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:01:19.044968229Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:01:39.895621577Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:01:39.902254246Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:01:49.05759451Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:01:49.064458524Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:02:19.084798681Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:02:19.091645616Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:02:39.910758436Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:02:39.917643015Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:02:49.10629544Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:02:49.113677555Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:03:19.129094944Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:03:19.13477181Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:03:39.93462799Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:03:39.944492791Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:03:49.141639052Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:03:49.147883007Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:04:19.16310336Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:04:19.168528563Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:04:39.956280905Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:04:39.964066905Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:04:49.176742928Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:04:49.182716594Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:05:19.20013428Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:05:19.205519386Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:05:39.975863761Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:05:39.983937282Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:05:49.219142048Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:05:49.225548295Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:06:19.241246584Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:06:19.246684948Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:06:39.996376749Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:06:40.004229156Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:06:49.270654776Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:06:49.281649884Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:07:19.300490357Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:07:19.30913545Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:07:40.015632459Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:07:40.023275236Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:07:49.325804064Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:07:49.33266351Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:08:19.351469289Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:08:19.358434623Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:08:40.035306418Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:08:40.042327673Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:08:49.372630246Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:08:49.378422927Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:09:19.397322039Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:09:19.403569302Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:09:40.055210075Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:09:40.063396982Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:09:49.414216002Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:09:49.419746076Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:10:19.440332499Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:10:19.446271893Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:10:40.078683044Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:10:40.087404613Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:10:49.456521988Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:10:49.462634841Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:11:19.479680527Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:11:19.487725687Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:11:40.09988396Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:11:40.10837872Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:11:49.503693272Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:11:49.509657867Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:12:19.531066317Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:12:19.539288875Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:12:40.12217075Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:12:40.129876546Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:12:49.556591516Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:12:49.564400071Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:13:19.582668182Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:13:19.588970842Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:13:40.141105608Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:13:40.14827557Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:13:49.61050754Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:13:49.619919837Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:14:19.6487239Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:14:19.654280469Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:14:40.161977308Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:14:40.171877857Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:14:49.666541455Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:14:49.67350053Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:15:19.691400855Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:15:19.698739675Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:15:40.182978842Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:15:40.190287474Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:15:49.710125027Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:15:49.716533728Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:16:19.736193833Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:16:19.742901878Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:16:40.203520524Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:16:40.210743714Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:16:49.754735224Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:16:49.760622439Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:17:19.775565582Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:17:19.784332257Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:17:40.223708296Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:17:40.230968793Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:17:49.802006541Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:17:49.808265053Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:18:19.822469063Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:18:19.829856893Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:18:40.24207568Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:18:40.250187013Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:18:49.839964601Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:18:49.845912376Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:19:19.861818073Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:19:19.868525792Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:19:40.260017977Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:19:40.267495507Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:19:49.881804323Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:19:49.889721035Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:20:19.900096575Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:20:19.906176659Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:20:40.28116577Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:20:40.288134868Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:20:49.922710722Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:20:49.929567597Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:21:19.948124065Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:21:19.955188922Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:21:40.298706398Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:21:40.306491385Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:21:49.972511116Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:21:49.981694261Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:22:19.998465704Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:22:20.006022915Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:22:40.317763856Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:22:40.326518032Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:22:50.024345879Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:22:50.030579026Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:23:20.042904725Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:23:20.048933901Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:23:40.336959263Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:23:40.344176674Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:23:50.064082847Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:23:50.069553029Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:24:20.090617708Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:24:20.098465217Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:24:40.356307043Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:24:40.36293565Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:24:50.11786046Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:24:50.126240796Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:25:20.140106131Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:25:20.14551994Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:25:40.373623356Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:25:40.381033992Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:25:50.153409509Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:25:50.159366513Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:26:20.177903617Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:26:20.18461315Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:26:40.393672601Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:26:40.400754379Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:26:50.197184765Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:26:50.202414352Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:27:20.215569875Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:27:20.223438175Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:27:40.41078912Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:27:40.417653468Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:27:50.230653921Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:27:50.236419751Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:28:20.245712776Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:28:20.251798894Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:28:40.42738227Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:28:40.435803558Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:28:50.260031158Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:28:50.272181014Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:29:20.282294058Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:29:20.288360344Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:29:40.446333774Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:29:40.454473063Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:29:50.295271088Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:29:50.300622694Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:30:20.312093374Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:30:20.317559196Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:30:40.471186364Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:30:40.485976261Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:30:50.324405366Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:30:50.329562828Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:31:20.338993214Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:31:20.345950336Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:31:40.499103698Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:31:40.507091948Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:31:50.353844263Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:31:50.36006056Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:32:20.37093192Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:32:20.377635543Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:32:40.517183105Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:32:40.526038943Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:32:50.387509293Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:32:50.394469014Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:33:20.40544814Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:33:20.411129444Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:33:40.538062563Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:33:40.54532305Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:33:50.418417212Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:33:50.424885721Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:34:20.434399927Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:34:20.440252874Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:34:40.555137146Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:34:40.563215073Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:34:50.447787446Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:34:50.453733931Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:35:20.464011028Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:35:20.470728137Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:35:40.574465864Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:35:40.582760524Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:35:50.478313415Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:35:50.48441402Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:36:20.494208095Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:36:20.500635425Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:36:40.593243109Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:36:40.601564696Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:36:50.508698974Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:36:50.516571736Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:37:20.530818562Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:37:20.541397801Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:37:40.613033392Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:37:40.621355702Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:37:50.54961538Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:37:50.55655969Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:38:20.56962547Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:38:20.579857062Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:38:40.630625779Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:38:40.638514927Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:38:50.592670245Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:38:50.612312895Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:39:20.623136653Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:39:20.629972941Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:39:40.650840169Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:39:40.66012772Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:39:50.643790414Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:39:50.654115459Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:40:20.664868641Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:40:20.670725917Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:40:40.668661876Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:40:40.675842362Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:40:50.678122607Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:40:50.684381368Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:41:20.697388523Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:41:20.703822606Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:41:40.718615385Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:41:40.726021977Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:41:50.714502739Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:41:50.72054487Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:42:20.729655627Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:42:20.735183774Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:42:40.737430454Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:42:40.745240208Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:42:50.742653618Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:42:50.748820138Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:43:20.757887551Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:43:20.76460776Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:43:40.756527832Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:43:40.7637332Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:43:50.78741481Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:43:50.799001779Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:44:20.811252181Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:44:20.819264086Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:44:40.773828869Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:44:40.780906985Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:44:50.826622984Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:44:50.832374186Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:45:20.851015246Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:45:20.8566347Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:45:40.795592241Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:45:40.805660201Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:45:50.863451081Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:45:50.871168393Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:46:20.88022205Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:46:20.886183874Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:46:40.817927199Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:46:40.826714122Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:46:50.893873378Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:46:50.899873438Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:47:20.909480271Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:47:20.915991062Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:47:40.840127716Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:47:40.847670101Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:47:50.92263716Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:47:50.92807942Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:48:20.971218453Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:48:21.006793652Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:48:40.860417947Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:48:40.86752486Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:48:51.017346095Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:48:51.023656362Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:49:21.033674335Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:49:21.041124215Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:49:40.880082928Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:49:40.888117212Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:49:51.048451133Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:49:51.054620845Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:50:21.064706942Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:50:21.071385277Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:50:40.899374198Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:50:40.908176546Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:50:51.080829233Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:50:51.087722178Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:51:21.098969531Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:51:21.112422702Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:51:40.91927374Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:51:40.926274002Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:51:51.122983692Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:51:51.130143372Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:52:21.140357342Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:52:21.146635107Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:52:40.940392225Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:52:40.947871558Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:52:51.155380122Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:52:51.161027858Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:53:21.171702222Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:53:21.177640778Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:53:40.961369654Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:53:40.969157113Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:53:51.19093694Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:53:51.199251103Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:54:21.211298604Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:54:21.218651436Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:54:40.978993833Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:54:40.986261326Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:54:51.227399232Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:54:51.23334918Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:55:21.242847202Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:55:21.248298014Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:55:40.99717054Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:55:41.00446139Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:55:51.258401876Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:55:51.2644654Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:56:21.273806838Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:56:21.279376261Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:56:41.016786373Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:56:41.024031588Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:56:51.288846671Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:56:51.294837501Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:57:21.303996593Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:57:21.310236004Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:57:41.034962041Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:57:41.041584463Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:57:51.319454201Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:57:51.325497291Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:58:21.334715486Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:58:21.341106745Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:58:41.054856089Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:58:41.062933856Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:58:51.350779646Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:58:51.35689453Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:59:21.36544961Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:59:21.371963974Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T08:59:41.073579333Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:59:41.080884649Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T08:59:51.381299244Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T08:59:51.387889009Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:00:21.396768127Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:00:21.402459038Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:00:41.093767648Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:00:41.103905354Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:00:51.413155093Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:00:51.41913818Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:01:21.428207567Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:01:21.434618949Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:01:41.114754752Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:01:41.122950948Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:01:51.444071953Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:01:51.450843625Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:02:21.461173332Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:02:21.467392796Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:02:41.132862385Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:02:41.139763486Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:02:51.474244236Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:02:51.479799788Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:03:21.488604751Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:03:21.494121108Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:03:41.150532226Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:03:41.157989139Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:03:51.502696556Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:03:51.50776602Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:04:21.517109602Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:04:21.522687493Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:04:41.167412673Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:04:41.176529725Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:04:51.530012982Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:04:51.535815325Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:05:21.546308441Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:05:21.551812965Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:05:41.186463649Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:05:41.194478869Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:05:51.559707391Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:05:51.566768176Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:06:21.578634938Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:06:21.585848597Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:06:41.208877573Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:06:41.217479293Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:06:51.597231625Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:06:51.608090895Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:07:21.617841462Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:07:21.623410242Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:07:41.231078716Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:07:41.239653078Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:07:51.637810824Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:07:51.64651079Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:08:21.656823055Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:08:21.665171348Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:08:41.2490394Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:08:41.255556955Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:08:51.674982985Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:08:51.680517389Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:09:21.689116244Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:09:21.695108656Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:09:41.266707613Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:09:41.273957546Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:09:51.705944118Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:09:51.714175016Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:10:21.726097372Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:10:21.73358877Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:10:41.284060382Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:10:41.292031009Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:10:51.746182065Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:10:51.755246969Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:11:21.765378765Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:11:21.771776153Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:11:41.304174459Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:11:41.312260302Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:11:51.78087181Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:11:51.786977667Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:12:21.803017274Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:12:21.808875621Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:12:41.322208887Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:12:41.330211473Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:12:51.819822914Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:12:51.825896172Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:13:21.834857027Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:13:21.840734511Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:13:41.341592125Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:13:41.349020652Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:13:51.853291435Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:13:51.860454603Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:14:21.869632366Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:14:21.875140626Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:14:41.362558421Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:14:41.370129096Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:14:51.88424657Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:14:51.892091846Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:15:21.902644074Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:15:21.908361173Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:15:41.38029056Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:15:41.388178613Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:15:51.919102951Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:15:51.924678324Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:16:21.935520164Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:16:21.942325052Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:16:41.401520139Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:16:41.408339776Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:16:51.953896667Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:16:51.962311868Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:17:21.972138683Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:17:21.978673397Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:17:41.425927353Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:17:41.4327173Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:17:51.988067464Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:17:51.993391423Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:18:22.002549412Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:18:22.008389358Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:18:41.447176592Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:18:41.455742397Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:18:52.01800745Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:18:52.025094469Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:19:22.03631706Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:19:22.042974411Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:19:41.465967508Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:19:41.472973431Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:19:52.052516549Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:19:52.058478069Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:20:22.068553681Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:20:22.074454317Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:20:41.488296654Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:20:41.495801737Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:20:52.084709455Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:20:52.090983482Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:21:22.101541255Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:21:22.108127959Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:21:41.507419419Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:21:41.514397201Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:21:52.120394345Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:21:52.126987103Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:22:22.135628675Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:22:22.141537288Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:22:41.530543371Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:22:41.539719146Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:22:52.151398929Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:22:52.156961062Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:23:22.16791601Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:23:22.173323097Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:23:41.550529204Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:23:41.557545274Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:23:52.184061528Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:23:52.193315343Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:24:22.204532207Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:24:22.21213598Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:24:41.573195158Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:24:41.580569901Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:24:52.222770154Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:24:52.229194971Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:25:22.245434754Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:25:22.25192519Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:25:41.593687569Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:25:41.602083387Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:25:52.262205032Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:25:52.269030028Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:26:22.278928741Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:26:22.285446109Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:26:41.616712993Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:26:41.624416274Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:26:52.298511672Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:26:52.30430194Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:27:22.316258475Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:27:22.322922171Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:27:41.63536079Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:27:41.643236783Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:27:52.334299393Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:27:52.339965161Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:28:22.351369932Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:28:22.358714866Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:28:41.653944469Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:28:41.662581474Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:28:52.370080121Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:28:52.37671779Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:29:22.385441421Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:29:22.392123379Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:29:41.676473464Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:29:41.684276383Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:29:52.400208539Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:29:52.406003407Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:30:22.415556345Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:30:22.421134608Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:30:41.702173515Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:30:41.710889714Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:30:52.430678512Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:30:52.43820329Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:31:22.447793649Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:31:22.453801314Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:31:41.723068113Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:31:41.730644579Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:31:52.463915081Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:31:52.470788227Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:32:22.481279164Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:32:22.487893257Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:32:41.7457042Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:32:41.752387465Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:32:52.5004062Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:32:52.508115111Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:33:22.518863501Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:33:22.524991433Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:33:41.763874246Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:33:41.770864269Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:33:52.535996472Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:33:52.542524299Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:34:22.55719183Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:34:22.565832574Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:34:41.782073619Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:34:41.790787662Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:34:52.57784854Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:34:52.584212728Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:35:22.597801163Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:35:22.604478324Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:35:41.802629031Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:35:41.810199818Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:35:52.617039372Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:35:52.622598697Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:36:22.632654884Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:36:22.639493651Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:36:41.822696919Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:36:41.831471082Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:36:52.650851504Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:36:52.657818015Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:37:22.669381683Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:37:22.675283639Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:37:41.841731864Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:37:41.848207894Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:37:52.685592494Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:37:52.691657811Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:38:22.700817509Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:38:22.706350146Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:38:41.863596346Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:38:41.872280404Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:38:52.716886735Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:38:52.723204461Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:39:22.733153713Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:39:22.738789423Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:39:41.882668886Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:39:41.891585202Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:39:52.748728686Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:39:52.754416936Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:40:22.765819092Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:40:22.772812603Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:40:41.899969923Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:40:41.90850314Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:40:52.784175883Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:40:52.789514998Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:41:22.801172822Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:41:22.807267407Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:41:41.918500243Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:41:41.926147982Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:41:52.818307979Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:41:52.824252603Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:42:22.832931678Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:42:22.83810474Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:42:41.934914366Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:42:41.942378877Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:42:52.847353764Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:42:52.852577321Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:43:22.863771284Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:43:22.869597644Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:43:41.953353657Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:43:41.961188407Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:43:52.882808362Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:43:52.88878611Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:44:22.901991575Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:44:22.907833718Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:44:41.979857376Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:44:41.989980055Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:44:52.918397405Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:44:52.923557947Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:45:22.932778288Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:45:22.939172994Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:45:42.003431766Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:45:42.010744309Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:45:52.949065072Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:45:52.954801119Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:46:22.966028095Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:46:22.971715624Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:46:42.019178601Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:46:42.027330727Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:46:52.982879257Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:46:52.9889622Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:47:23.000724977Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:47:23.006705785Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:47:42.037799036Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:47:42.045094791Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:47:53.016761346Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:47:53.022127211Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:48:23.032861427Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:48:23.039129312Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:48:42.054890513Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:48:42.061973151Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:48:53.04760738Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:48:53.053592263Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:49:23.062335709Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:49:23.067796398Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:49:42.07325746Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:49:42.080284528Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:49:53.081408208Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:49:53.087323027Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:50:23.101034581Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:50:23.109000444Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:50:42.093611386Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:50:42.105804109Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:50:53.121275816Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:50:53.127337594Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:51:23.135485059Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:51:23.141066255Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:51:42.117417016Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:51:42.127978134Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:51:53.15257427Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:51:53.158312898Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:52:23.167414981Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:52:23.173309344Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:52:42.146412131Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:52:42.158479861Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:52:53.183411305Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:52:53.190135102Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:53:23.206630306Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:53:23.214412518Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:53:42.172725891Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:53:42.18137992Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:53:53.224168538Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:53:53.230088902Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:54:23.238499921Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:54:23.244692675Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:54:42.192223481Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:54:42.198941515Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:54:53.255885571Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:54:53.261338359Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:55:23.272712532Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:55:23.27938582Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:55:42.210335252Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:55:42.216881055Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:55:53.290682662Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:55:53.296783852Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:56:23.306897246Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:56:23.312853867Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:56:42.229310072Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:56:42.238855671Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:56:53.32332127Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:56:53.328947682Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:57:23.340394821Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:57:23.346279315Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:57:42.252701918Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:57:42.262939406Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:57:53.360971005Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:57:53.369274891Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:58:23.382011039Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:58:23.388920812Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:58:42.272737188Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:58:42.280535316Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:58:53.398296938Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:58:53.403800179Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:59:23.412629176Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:59:23.418237287Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T09:59:42.291961581Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:59:42.300949193Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T09:59:53.429198382Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T09:59:53.436379332Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:00:23.44942421Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:00:23.456575073Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:00:42.316407895Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:00:42.325187891Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:00:53.466923662Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:00:53.472399425Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:01:23.484712427Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:01:23.491701577Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:01:42.346773313Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:01:42.354083521Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:01:53.50489851Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:01:53.512198471Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:02:23.525314064Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:02:23.533865617Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:02:42.36438456Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:02:42.371939178Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:02:53.544704634Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:02:53.550952137Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:03:23.562681205Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:03:23.568857452Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:03:42.383268875Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:03:42.389987346Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:03:53.582302101Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:03:53.588446638Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:04:23.600195461Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:04:23.60708157Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:04:42.399561441Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:04:42.406131189Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:04:53.616885693Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:04:53.622961964Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:05:23.63291056Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:05:23.638603866Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:05:42.417009779Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:05:42.425366228Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:05:53.649954013Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:05:53.655406648Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:06:23.667358163Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:06:23.672950052Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:06:42.440532352Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:06:42.446898716Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:06:53.687610891Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:06:53.695979681Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:07:23.707199114Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:07:23.713860232Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:07:42.457879786Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:07:42.465097431Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:07:53.724279832Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:07:53.730124485Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:08:23.740723096Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:08:23.74850279Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:08:42.479163067Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:08:42.486251621Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:08:53.761853743Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:08:53.767695869Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:09:23.778348137Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:09:23.784343835Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:09:42.503725448Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:09:42.513613022Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:09:53.794579387Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:09:53.799998305Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:10:23.81100473Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:10:23.81694344Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:10:42.523730433Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:10:42.532434057Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:10:53.82656806Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:10:53.832124327Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:11:23.843470515Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:11:23.849097452Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:11:42.545483573Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:11:42.553217377Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:11:53.858963451Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:11:53.864546726Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:12:23.877806031Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:12:23.884035443Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:12:42.564566005Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:12:42.574644592Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:12:53.896938863Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:12:53.902418147Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:13:23.914676829Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:13:23.92112802Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:13:42.585852451Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:13:42.592932775Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:13:53.932089517Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:13:53.937578449Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:14:23.949315319Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:14:23.95576749Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:14:42.606079266Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:14:42.614083652Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:14:53.969212544Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:14:53.975071218Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:15:23.988375915Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:15:23.994740641Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:15:42.626818071Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:15:42.635114952Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:15:54.011253472Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:15:54.02569544Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:16:24.039028161Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:16:24.045685651Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:16:42.646903189Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:16:42.654909867Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:16:54.055978289Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:16:54.062233584Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:17:24.07359863Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:17:24.079685058Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:17:42.666599376Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:17:42.673664811Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:17:54.09029337Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:17:54.097266919Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:18:24.111181224Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:18:24.120344913Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:18:42.683359548Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:18:42.690233718Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:18:54.13207171Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:18:54.137857628Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:19:24.150579008Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:19:24.156359433Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:19:42.702003495Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:19:42.70955311Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:19:54.168312584Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:19:54.173939124Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:20:24.18401381Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:20:24.190276112Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:20:42.721940182Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:20:42.730093568Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:20:54.207356937Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:20:54.216451846Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:21:24.228643108Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:21:24.236819428Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:21:42.742245697Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:21:42.750524218Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:21:54.249879999Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:21:54.256278168Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:22:24.267471006Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:22:24.273506722Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:22:42.767494959Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:22:42.774798908Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:22:54.285455617Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:22:54.290801341Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:23:24.303285417Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:23:24.310375494Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:23:42.7875357Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:23:42.794528818Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:23:54.323157379Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:23:54.328813401Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:24:24.338715019Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:24:24.344506054Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:24:42.810294228Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:24:42.818011891Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:24:54.359338604Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:24:54.369397498Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:25:24.380859735Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:25:24.386279003Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:25:42.827483305Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:25:42.835970017Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:25:54.398467007Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:25:54.404696528Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:26:24.418249212Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:26:24.423818286Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:26:42.846586999Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:26:42.853438548Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:26:54.433434288Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:26:54.439732339Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:27:24.449367137Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:27:24.455929571Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:27:42.870284353Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:27:42.878138966Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:27:54.469762166Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:27:54.478486991Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:28:24.489137655Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:28:24.495319144Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:28:42.890476786Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:28:42.897947126Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:28:54.507944837Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:28:54.513876945Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:29:24.526540864Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:29:24.532600785Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:29:42.910616634Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:29:42.91749027Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:29:54.542687243Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:29:54.54830451Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:30:24.557119557Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:30:24.564823251Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:30:42.946013196Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:30:42.956617911Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:30:54.576842379Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:30:54.582455905Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:31:24.594288287Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:31:24.601745371Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:31:42.972791161Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:31:42.985823604Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:31:54.615954884Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:31:54.621607123Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:32:24.633627671Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:32:24.639422688Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:32:43.006590461Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:32:43.017248153Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:32:54.649474373Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:32:54.654768578Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:33:24.664342816Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:33:24.670261341Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:33:43.028604404Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:33:43.035869067Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:33:54.683732585Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:33:54.68991207Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:34:24.700219985Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:34:24.706440641Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:34:43.053657328Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:34:43.064590031Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:34:54.71873565Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:34:54.724583247Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:35:24.736228008Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:35:24.741458213Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:35:43.079882058Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:35:43.091515691Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:35:54.753540357Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:35:54.760393059Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:36:24.769456704Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:36:24.77496736Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:36:43.108181487Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:36:43.116810452Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:36:54.786666728Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:36:54.792479712Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:37:24.802072683Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:37:24.809512679Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:37:43.130803876Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:37:43.13979228Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:37:54.821240522Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:37:54.827576261Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:38:24.838658303Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:38:24.843818079Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:38:43.151973404Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:38:43.160620806Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:38:54.857494305Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:38:54.865254029Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:39:24.876444951Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:39:24.882752737Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:39:43.172691621Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:39:43.179610096Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:39:54.894110721Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:39:54.900371858Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:40:24.914575884Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:40:24.920305275Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:40:43.192739334Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:40:43.203736834Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:40:54.934104398Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:40:54.943997568Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:41:24.957519185Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:41:24.963565752Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:41:43.21758119Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:41:43.226622266Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:41:54.976799216Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:41:54.982645952Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:42:24.995398965Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:42:25.00077762Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:42:43.242885033Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:42:43.250729141Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:42:55.034796234Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:42:55.041639785Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:43:25.052933111Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:43:25.060389758Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:43:43.26183108Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:43:43.26843493Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:43:55.069928352Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:43:55.077869358Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:44:25.088365167Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:44:25.09423176Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:44:43.280916718Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:44:43.287917932Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:44:55.10747182Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:44:55.114580416Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:45:25.12495342Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:45:25.130381616Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:45:43.300948686Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:45:43.307788941Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:45:55.144535303Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:45:55.150681233Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:46:25.162421078Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:46:25.168197627Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:46:43.326347529Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:46:43.335034875Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:46:55.179467945Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:46:55.184778468Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:47:25.196446789Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:47:25.202590381Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:47:43.34873707Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:47:43.356888761Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:47:55.216376428Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:47:55.223623956Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:48:25.238225315Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:48:25.244845544Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:48:43.367715083Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:48:43.374711964Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:48:55.257889511Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:48:55.263562573Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:49:25.273149268Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:49:25.279070374Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:49:43.38625207Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:49:43.393503905Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:49:55.289936937Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:49:55.295875193Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:50:25.308540888Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:50:25.314104208Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:50:43.405574072Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:50:43.41299519Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:50:55.329860161Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:50:55.338280334Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:51:25.353952296Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:51:25.35960062Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:51:43.426393758Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:51:43.435783202Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:51:55.371710419Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:51:55.378142072Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:52:25.388404221Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:52:25.394141771Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:52:43.446802977Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:52:43.455198335Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:52:55.411615291Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:52:55.419505166Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:53:25.434364781Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:53:25.440551851Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:53:43.467629572Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:53:43.475003272Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:53:55.455246768Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:53:55.46215695Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:54:25.475597352Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:54:25.482679834Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:54:43.492194305Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:54:43.501095914Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:54:55.496857822Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:54:55.502758201Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:55:25.512315328Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:55:25.519084392Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:55:43.512615073Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:55:43.519682667Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:55:55.533015326Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:55:55.539371115Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:56:25.55297107Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:56:25.559586282Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:56:43.529780196Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:56:43.537259465Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:56:55.575342186Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:56:55.582054547Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:57:25.594876253Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:57:25.601148623Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:57:43.548645669Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:57:43.555629897Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:57:55.614303921Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:57:55.619554683Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:58:25.633210277Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:58:25.639390731Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:58:43.569137731Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:58:43.577142858Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:58:55.653217054Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:58:55.65920453Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:59:25.673427707Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:59:25.678959147Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T10:59:43.589826608Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:59:43.597328806Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T10:59:55.694104634Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T10:59:55.700672876Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:00:25.715583474Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:00:25.729488735Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:00:43.614395898Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:00:43.624336594Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:00:55.744100446Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:00:55.752742163Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:01:25.765664744Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:01:25.772572265Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:01:43.656829198Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:01:43.672897318Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:01:55.785916142Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:01:55.792648479Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:02:25.809523878Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:02:25.816664466Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:02:43.686651623Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:02:43.695269885Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:02:55.834595058Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:02:55.8411287Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:03:25.853102211Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:03:25.85971823Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:03:43.708311348Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:03:43.718359636Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:03:55.868609344Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:03:55.874491222Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:04:25.886296496Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:04:25.89173789Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:04:43.735226454Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:04:43.74183678Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:04:55.905906164Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:04:55.911757401Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:05:25.92344852Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:05:25.929945468Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:05:43.753111339Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:05:43.760451611Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:05:55.941984495Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:05:55.949666088Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:06:25.963099787Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:06:25.968675062Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:06:43.774026071Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:06:43.780968658Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:06:55.98096404Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:06:55.986706763Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:07:26.003406532Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:07:26.009871154Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:07:43.793773784Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:07:43.803637867Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:07:56.021621426Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:07:56.029925238Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:08:26.044243222Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:08:26.052523188Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:08:43.816134147Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:08:43.8228875Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:08:56.070525591Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:08:56.079009621Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:09:26.094878187Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:09:26.101352396Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:09:43.836453536Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:09:43.843784632Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:09:56.113299114Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:09:56.120130523Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:10:26.131843816Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:10:26.137088541Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:10:43.860825468Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:10:43.868315205Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:10:56.151411902Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:10:56.156893929Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:11:26.170734756Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:11:26.177208961Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:11:43.882574249Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:11:43.891503599Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:11:56.186424178Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:11:56.192964647Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:12:26.203255279Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:12:26.20945265Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:12:43.909422843Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:12:43.916481407Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:12:56.221878469Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:12:56.227264633Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:13:26.241941136Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:13:26.24910371Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:13:43.930652432Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:13:43.937785023Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:13:56.258484358Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:13:56.263715146Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:14:26.278621996Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:14:26.285381409Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:14:43.955898927Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:14:43.964265588Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:14:56.297364957Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:14:56.303024638Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:15:26.31627874Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:15:26.323169452Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:15:43.976913899Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:15:43.985452389Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:15:56.334504343Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:15:56.340841574Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:16:26.352133484Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:16:26.357448674Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:16:44.002178877Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:16:44.009650247Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:16:56.369560944Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:16:56.375247673Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:17:26.389667115Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:17:26.395231708Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:17:44.021493622Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:17:44.029104687Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:17:56.409494971Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:17:56.415249591Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:18:26.429776657Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:18:26.435135997Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:18:44.04446103Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:18:44.052115658Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:18:56.444411556Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:18:56.449525813Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:19:26.459350676Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:19:26.464456199Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:19:44.063646323Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:19:44.071221845Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:19:56.478983278Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:19:56.484925009Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:20:26.500028625Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:20:26.506990126Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:20:44.087941741Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:20:44.095515604Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:20:56.519794975Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:20:56.525153466Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:21:26.54050948Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:21:26.546537944Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:21:44.111496364Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:21:44.119609149Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:21:56.56815588Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:21:56.577448772Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:22:26.594271602Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:22:26.609107921Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:22:44.138017249Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:22:44.145872446Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:22:56.62183124Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:22:56.628066845Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:23:26.63637614Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:23:26.642770336Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:23:44.160460595Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:23:44.17338588Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:23:56.657330917Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:23:56.663738455Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:24:26.680290861Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:24:26.68657231Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:24:44.18617251Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:24:44.193657123Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:24:56.700387533Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:24:56.706015372Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:25:26.715489962Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:25:26.723255404Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:25:44.205036563Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:25:44.213368977Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:25:56.731164397Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:25:56.736719495Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:26:26.751559557Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:26:26.757465906Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:26:44.232552217Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:26:44.24227752Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:26:56.769390064Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:26:56.775232755Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:27:26.793738775Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:27:26.801509178Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:27:44.254498267Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:27:44.260800327Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:27:56.813716829Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:27:56.819654478Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:28:26.835974944Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:28:26.842358587Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:28:44.270889263Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:28:44.277947135Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:28:56.850290889Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:28:56.857001502Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:29:26.873841226Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:29:26.879849973Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:29:44.287851063Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:29:44.295141061Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:29:56.894412276Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:29:56.900537007Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:30:26.920445927Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:30:26.925992856Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:30:44.306260873Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:30:44.312823838Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:30:56.937314603Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:30:56.943791903Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:31:26.96543931Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:31:26.970688223Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:31:44.324391062Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:31:44.331137055Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:31:56.984515237Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:31:56.990590398Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:32:27.005989407Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:32:27.01195731Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:32:44.346654907Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:32:44.354751701Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:32:57.023654617Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:32:57.030069347Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:33:27.042707351Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:33:27.050993863Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:33:44.371300562Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:33:44.378348568Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:33:57.061773539Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:33:57.068363536Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:34:27.085078817Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:34:27.090807425Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:34:44.393804534Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:34:44.401259883Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:34:57.102666246Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:34:57.109147308Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:35:27.124126374Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:35:27.130130443Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:35:44.413191514Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:35:44.420793359Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:35:57.146246136Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:35:57.153115077Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:36:27.170170161Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:36:27.175412059Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:36:44.435740904Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:36:44.443944952Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:36:57.183658499Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:36:57.18917854Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:37:27.201928037Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:37:27.207386559Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:37:44.46976551Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:37:44.477108491Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:37:57.222206894Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:37:57.227608924Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:38:27.243510123Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:38:27.24891156Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:38:44.493132689Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:38:44.499554938Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:38:57.260796849Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:38:57.267489611Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:39:27.279399807Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:39:27.284835194Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:39:44.510307273Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:39:44.517796046Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:39:57.302325999Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:39:57.309714539Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:40:27.325469555Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:40:27.332498019Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:40:44.532183228Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:40:44.539830589Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:40:57.347984774Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:40:57.35759104Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:41:27.372624586Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:41:27.37789216Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:41:44.551566499Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:41:44.558860331Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:41:57.38795277Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:41:57.394140994Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:42:27.409683476Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:42:27.417422505Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:42:44.574928834Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:42:44.583489404Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:42:57.430942868Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:42:57.437195243Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:43:27.452687724Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:43:27.458905905Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:43:44.595437262Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:43:44.602655943Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:43:57.484917789Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:43:57.498036675Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:44:27.513734305Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:44:27.519145312Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:44:44.617770925Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:44:44.625657212Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:44:57.528912601Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:44:57.535116701Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:45:27.545668141Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:45:27.551195549Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:45:44.637885489Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:45:44.645069314Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:45:57.566760665Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:45:57.57385712Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:46:27.597198855Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:46:27.607024722Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:46:44.657333308Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:46:44.664200945Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:46:57.618978468Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:46:57.625989585Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:47:27.634622371Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:47:27.640017515Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:47:44.678121607Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:47:44.684293702Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:47:57.659340367Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:47:57.668160332Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:48:27.684202818Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:48:27.689674677Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:48:44.697171969Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:48:44.702734915Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:48:57.703823109Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:48:57.709664542Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:49:27.723475919Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:49:27.729573989Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:49:44.713501008Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:49:44.719422005Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:49:57.745461675Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:49:57.752503075Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:50:27.776851023Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:50:27.803800358Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:50:44.73436038Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:50:44.741895302Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:50:57.821578653Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:50:57.829752146Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:51:27.844481925Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:51:27.849540422Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:51:44.756611553Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:51:44.764997216Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:51:57.864965204Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:51:57.872505911Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:52:27.887084992Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:52:27.892198312Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:52:44.779965237Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:52:44.785547293Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:52:57.906756284Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:52:57.91188982Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:53:27.926092265Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:53:27.931760646Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:53:44.803215833Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:53:44.814597518Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:53:57.946438762Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:53:57.953614935Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:54:27.970304518Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:54:27.979392107Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:54:44.826710671Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:54:44.832346298Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:54:57.99485815Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:54:58.00130634Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:55:28.017146153Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:55:28.025911389Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:55:44.841942858Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:55:44.847797763Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:55:58.041687219Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:55:58.047991523Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:56:28.062967726Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:56:28.068832945Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:56:44.857987363Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:56:44.863656938Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:56:58.083539637Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:56:58.090647043Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:57:28.108835545Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:57:28.117954319Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:57:44.918387773Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:57:44.930387884Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:57:58.134463395Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:57:58.140933934Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:58:28.154522585Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:58:28.16109082Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:58:44.948742855Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:58:44.95586729Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:58:58.177168058Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:58:58.183539192Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:59:28.196437173Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:59:28.203760487Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T11:59:44.969184331Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:59:44.977415521Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T11:59:58.218849378Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T11:59:58.226679146Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:00:28.242218874Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:00:28.248354759Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:00:44.994740015Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:00:45.001068128Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:00:58.262026832Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:00:58.269500064Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:01:28.280952594Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:01:28.289185449Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:01:45.013682724Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:01:45.021278122Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:01:58.306710217Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:01:58.315751565Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:02:28.33255365Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:02:28.340767393Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:02:45.038572848Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:02:45.044210329Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:02:58.359537522Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:02:58.369681117Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:03:28.379312167Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:03:28.385577226Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:03:45.055340303Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:03:45.061200608Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:03:58.407908897Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:03:58.416565566Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:04:28.432772883Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:04:28.438659028Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:04:45.078127505Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:04:45.086268116Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:04:58.456074627Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:04:58.462111974Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:05:28.478432984Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:05:28.486171905Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:05:45.097740172Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:05:45.104153967Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:05:58.5030642Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:05:58.512374704Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:06:28.52872272Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:06:28.535803906Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:06:45.123774103Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:06:45.129985159Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:06:58.544618767Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:06:58.550119348Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:07:28.562888771Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:07:28.571660353Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:07:45.140919699Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:07:45.147760078Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:07:58.591509556Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:07:58.599496755Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:08:28.619319455Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:08:28.626272316Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:08:45.163316732Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:08:45.171690895Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:08:58.643376609Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:08:58.649994194Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:09:28.668411727Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:09:28.676920725Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:09:45.181905025Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:09:45.190112228Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:09:58.686473546Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:09:58.69225567Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:10:28.710081504Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:10:28.717021934Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:10:45.205440624Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:10:45.211733698Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:10:58.734098054Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:10:58.739576985Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:11:28.754616745Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:11:28.759819911Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:11:45.225472847Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:11:45.231974782Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:11:58.776953916Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:11:58.782901477Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:12:28.797364653Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:12:28.803977855Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:12:45.249726717Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:12:45.256019221Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:12:58.817005745Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:12:58.823055152Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:13:28.837811114Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:13:28.843235912Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:13:45.266088519Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:13:45.273035605Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:13:58.855478757Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:13:58.864248131Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:14:28.880510737Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:14:28.889544623Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:14:45.291692535Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:14:45.298508099Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:14:58.90843814Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:14:58.915310308Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:15:28.932014309Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:15:28.937873499Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:15:45.312655775Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:15:45.31935469Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:15:58.956294224Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:15:58.962143373Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:16:28.979806753Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:16:28.986420525Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:16:45.330654881Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:16:45.336429104Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:16:59.002364247Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:16:59.008704546Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:17:29.018457578Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:17:29.023663507Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:17:45.35175175Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:17:45.36083783Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:17:59.034749746Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:17:59.039763009Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:18:29.053740321Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:18:29.060556842Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:18:45.373428817Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:18:45.380883596Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:18:59.073514745Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:18:59.079231494Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:19:29.094891182Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:19:29.104859187Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:19:45.391111598Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:19:45.397002719Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:19:59.121845729Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:19:59.127710886Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:20:29.147162613Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:20:29.154115056Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:20:45.414471299Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:20:45.420782062Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:20:59.173475201Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:20:59.179186558Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:21:29.192882898Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:21:29.199696631Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:21:45.431668914Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:21:45.43747585Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:21:59.213740095Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:21:59.220936061Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:22:29.239004778Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:22:29.244289554Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:22:45.4549746Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:22:45.461009578Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:22:59.260365817Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:22:59.265880872Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:23:29.286150162Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:23:29.29259902Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:23:45.471531852Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:23:45.47937724Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:23:59.310896699Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:23:59.315872103Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:24:29.333825631Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:24:29.341620535Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:24:45.498091201Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:24:45.503870727Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:24:59.362122505Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:24:59.370673965Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:25:29.391340847Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:25:29.398391537Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:25:45.515572345Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:25:45.523661128Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:25:59.414186259Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:25:59.419736279Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:26:29.43634986Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:26:29.441323693Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:26:45.534818632Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:26:45.54119968Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:26:59.45683524Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:26:59.463137335Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:27:29.480605571Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:27:29.486942024Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:27:45.552426595Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:27:45.55968187Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:27:59.503275823Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:27:59.509033308Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:28:29.525689852Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:28:29.532853268Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:28:45.570023596Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:28:45.581928931Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:28:59.545004018Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:28:59.551484376Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:29:29.570333165Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:29:29.578271734Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:29:45.591744071Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:29:45.598584455Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:29:59.596338362Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:29:59.601894854Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:30:29.620000106Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:30:29.627523176Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:30:45.617009753Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:30:45.623606102Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:30:59.644771301Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:30:59.650339752Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:31:29.669052672Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:31:29.675618171Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:31:45.634260235Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:31:45.640139321Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:31:59.686391441Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:31:59.691483216Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:32:29.707002243Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:32:29.714201724Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:32:45.657699688Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:32:45.664339981Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:32:59.732316763Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:32:59.738538005Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:33:29.754620278Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:33:29.76080932Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:33:45.675561241Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:33:45.681540306Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:33:59.777277349Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:33:59.783274919Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:34:29.840824692Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:34:29.860332126Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:34:45.70175996Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:34:45.708310057Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:34:59.880750902Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:34:59.889925253Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:35:29.907082183Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:35:29.912495537Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:35:45.718830003Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:35:45.725695219Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:35:59.931115016Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:35:59.936643833Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:36:29.954218678Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:36:29.960255604Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:36:45.741581091Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:36:45.74916655Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:36:59.97610229Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:36:59.982144573Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:37:29.997920062Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:37:30.003099249Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:37:45.758303021Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:37:45.764688533Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:38:00.01995386Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:38:00.025214827Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:38:30.044260571Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:38:30.051372832Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:38:45.780611903Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:38:45.789762507Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:39:00.069290675Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:39:00.076088751Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:39:30.092364837Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:39:30.100137452Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:39:45.80083174Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:39:45.807830609Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:40:00.116407842Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:40:00.124399296Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:40:30.143557274Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:40:30.150682881Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:40:45.818027687Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:40:45.828223645Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:41:00.170001483Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:41:00.178248698Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:41:30.195648583Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:41:30.202244099Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:41:45.839799326Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:41:45.847360116Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:42:00.220512169Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:42:00.228016699Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:42:30.244172382Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:42:30.249700098Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:42:45.859589766Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:42:45.867414474Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:43:00.277176784Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:43:00.283397129Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:43:30.301793996Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:43:30.308095259Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:43:45.88040851Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:43:45.887883206Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:44:00.326382406Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:44:00.333784345Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:44:30.351661462Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:44:30.35838137Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:44:45.902213593Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:44:45.909244485Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:45:00.379515276Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:45:00.388401674Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:45:30.409879864Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:45:30.41710076Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:45:45.920463845Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:45:45.928118642Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:46:00.434326087Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:46:00.440232509Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:46:30.458361304Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:46:30.464938143Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:46:45.9371274Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:46:45.944021664Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:47:00.481626838Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:47:00.487997609Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:47:30.50797256Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:47:30.515324441Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:47:45.95479513Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:47:45.961751511Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:48:00.533478111Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:48:00.538981519Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:48:30.55802085Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:48:30.564592434Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:48:45.971554408Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:48:45.978281421Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:49:00.583264663Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:49:00.589656857Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:49:30.601874187Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:49:30.609064628Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:49:45.989518978Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:49:45.995930951Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:50:00.623824099Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:50:00.629137485Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:50:30.643372506Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:50:30.649516687Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:50:46.016479225Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:50:46.024328467Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:51:00.662134525Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:51:00.668295807Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:51:30.681080162Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:51:30.686948521Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:51:46.036869824Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:51:46.043825435Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:52:00.704303247Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:52:00.709484513Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:52:30.72703409Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:52:30.733475644Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:52:46.056377663Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:52:46.063541251Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:53:00.745081883Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:53:00.750557484Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:53:30.76695042Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:53:30.77233882Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:53:46.074028958Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:53:46.081857963Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:54:00.784180501Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:54:00.790876639Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:54:30.808661448Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:54:30.814749991Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:54:46.095210604Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:54:46.103029674Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:55:00.833418319Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:55:00.838869117Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:55:30.859255081Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:55:30.866588475Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:55:46.114913401Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:55:46.121589364Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:56:00.884512966Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:56:00.890684058Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:56:30.909218564Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:56:30.914936038Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:56:46.136105437Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:56:46.142599718Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:57:00.924733849Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:57:00.931126516Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:57:30.949616014Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:57:30.955161194Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:57:46.152444661Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:57:46.159500826Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:58:00.963466468Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:58:00.969429188Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:58:30.987794264Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:58:30.993550945Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:58:46.168180098Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:58:46.174800242Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:59:01.010474027Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:59:01.016332788Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:59:31.033474951Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T12:59:31.039392247Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T12:59:46.18440888Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T12:59:46.190996421Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:00:01.058435392Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:00:01.065529097Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:00:31.086767493Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:00:31.093151807Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:00:46.206101553Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:00:46.213404438Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:01:01.112774182Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:01:01.119794962Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:01:31.128453826Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:01:31.134314599Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:01:46.223719789Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:01:46.229979208Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:02:01.149747644Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:02:01.155562985Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:02:31.17322604Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:02:31.180355723Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:02:46.240429417Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:02:46.246905405Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:03:01.198729784Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:03:01.204789564Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:03:31.224001936Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:03:31.229613852Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:03:46.258308987Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:03:46.265703716Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:04:01.247764373Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:04:01.25362089Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:04:31.276558946Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:04:31.28601089Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:04:46.282598646Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:04:46.28940567Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:05:01.293846427Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:05:01.299814934Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:05:31.316522022Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:05:31.322619501Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:05:46.300036108Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:05:46.307620257Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:06:01.340865257Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:06:01.345933747Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:06:31.366003345Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:06:31.37131082Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:06:46.331356388Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:06:46.341452047Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:07:01.387103443Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:07:01.393155338Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:07:31.411531886Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:07:31.41714657Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:07:46.354579926Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:07:46.363641715Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:08:01.433938328Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:08:01.440215757Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:08:31.450880135Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:08:31.457320478Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:08:46.382427212Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:08:46.389839933Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:09:01.473553882Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:09:01.479626247Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:09:31.502886512Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:09:31.510294132Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:09:46.402343314Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:09:46.410246368Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:10:01.530169686Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:10:01.536616646Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:10:31.548404066Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:10:31.557383083Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:10:46.426485439Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:10:46.433016545Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:11:01.58533639Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:11:01.594627182Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:11:31.60581044Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:11:31.615425021Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:11:46.445464841Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:11:46.452545025Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:12:01.634544333Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:12:01.639568697Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:12:31.65262083Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:12:31.65873202Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:12:46.478663193Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=warn caller=logger.go:55 ts=2024-07-12T13:12:46.484534269Z component=forwarder/worker msg="Failed to report status" err="failed to get ObservabilityAddon open-cluster-management-addon-observability/observability-addon: Get \"https://[fd02::1]:443/apis/observability.open-cluster-management.io/v1beta1/namespaces/open-cluster-management-addon-observability/observabilityaddons/observability-addon\": dial tcp [fd02::1]:443: connect: connection refused"
level=error caller=logger.go:60 ts=2024-07-12T13:12:46.484593631Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:13:01.675504809Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:13:01.682127673Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:13:31.708141203Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:13:31.715723387Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:13:46.495784692Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:13:46.515655619Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:14:01.736359511Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:14:01.746025856Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:14:31.76691111Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:14:31.774597713Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:14:46.530344984Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:14:46.537092296Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:15:01.793820171Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:15:01.799466738Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:15:31.82331136Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:15:31.829521144Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:15:46.549060769Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:15:46.556836418Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:16:01.847726773Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:16:01.853462222Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:16:31.873487715Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:16:31.878542265Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:16:46.570674736Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:16:46.577017741Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:17:01.894685309Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:17:01.901652703Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:17:31.920421241Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:17:31.925578834Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:17:46.590096917Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:17:46.597552801Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:18:01.9418151Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:18:01.948114338Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:18:31.966442872Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:18:31.971495086Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:18:46.618388722Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:18:46.628784152Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:19:01.987172538Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:19:01.993315469Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:19:32.012783549Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:19:32.018916497Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:19:46.639167154Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:19:46.645843912Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:20:02.037492498Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:20:02.04231721Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:20:32.062400325Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:20:32.068623706Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:20:46.658779109Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:20:46.66620147Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:21:02.09113627Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:21:02.103808645Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:21:32.122091799Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:21:32.12815495Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:21:46.67684036Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:21:46.685517161Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:22:02.147718666Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:22:02.153752743Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:22:32.175488587Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:22:32.181745552Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:22:46.704605827Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:22:46.711847444Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:23:02.194102555Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:23:02.200531176Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:23:32.209798153Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:23:32.215357328Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:23:46.72445773Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:23:46.731847348Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:24:02.233952639Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:24:02.239561194Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:24:32.259691306Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:24:32.265754061Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:24:46.751751759Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:24:46.75843346Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:25:02.2808441Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:25:02.287386454Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:25:32.309123755Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:25:32.314767551Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:25:46.768947113Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:25:46.776584325Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:26:02.325690859Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:26:02.3319798Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:26:32.342338802Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:26:32.349344848Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:26:46.795110686Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:26:46.802382225Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:27:02.358768997Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:27:02.364767196Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:27:32.383736522Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:27:32.389561572Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:27:46.814493915Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:27:46.822115986Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:28:02.400271735Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:28:02.405573811Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:28:32.425542124Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:28:32.432379737Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:28:46.835998049Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:28:46.843491712Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:29:02.452547573Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:29:02.458419253Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:29:32.481183968Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:29:32.490572329Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:29:46.857413294Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:29:46.864884505Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:30:02.511272743Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:30:02.51782397Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:30:32.537147504Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:30:32.543322531Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:30:46.884714285Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:30:46.891514983Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:31:02.562022812Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:31:02.568422486Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:31:32.588031323Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:31:32.593612975Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:31:46.905256682Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:31:46.913914523Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:32:02.616881713Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:32:02.627644208Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:32:32.647158827Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:32:32.653156402Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:32:46.933752435Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:32:46.941249112Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:33:02.675989111Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:33:02.682297325Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:33:32.702558791Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:33:32.708371463Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:33:46.956031598Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:33:46.963366234Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:34:02.722288798Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:34:02.727835931Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:34:32.750167991Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:34:32.760628515Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:34:46.980731213Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:34:46.98727548Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:35:02.781477874Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:35:02.787216964Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:35:32.806331582Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:35:32.813279573Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:35:46.997399199Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:35:47.004988583Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:36:02.833820669Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:36:02.839886231Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:36:32.85853465Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:36:32.864236651Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:36:47.017105596Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:36:47.024384034Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:37:02.884412139Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:37:02.891715972Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:37:32.909922431Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:37:32.915553461Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:37:47.035626046Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:37:47.04227108Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:38:02.930954475Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:38:02.937097878Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:38:32.95833604Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:38:32.965847984Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:38:47.058929652Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:38:47.067102201Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:39:02.991463811Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:39:02.997132288Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:39:33.00840403Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:39:33.015751499Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:39:47.078604335Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:39:47.08625711Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:40:03.037251499Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:40:03.045792419Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:40:33.065660573Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:40:33.071993493Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:40:47.106857389Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:40:47.11820896Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:41:03.083982809Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:41:03.090109686Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:41:33.108062509Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:41:33.114917371Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:41:47.133528416Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:41:47.142314188Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:42:03.133499945Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:42:03.139332858Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:42:33.157331829Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:42:33.163144001Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:42:47.151815354Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:42:47.158569743Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:43:03.17960985Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:43:03.185664015Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:43:33.203928695Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:43:33.209919045Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:43:47.175290816Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:43:47.188502749Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:44:03.227764418Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:44:03.233643746Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:44:33.254840128Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:44:33.261209296Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:44:47.208009618Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:44:47.218427211Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:45:03.26995573Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:45:03.275350154Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:45:33.295881847Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:45:33.301199411Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:45:47.230483107Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:45:47.238858356Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:46:03.316852956Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:46:03.32293886Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:46:33.336034019Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:46:33.342410492Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:46:47.260761469Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:46:47.271245762Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:47:03.362805739Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:47:03.369941138Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:47:33.394173202Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:47:33.401736858Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:47:47.283186754Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:47:47.290108904Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:48:03.411741411Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:48:03.417841577Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:48:33.425604989Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:48:33.431190913Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:48:47.309416271Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:48:47.320522259Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:49:03.44861625Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:49:03.455118564Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:49:33.464091495Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:49:33.469979225Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:49:47.33708696Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:49:47.345556069Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:50:03.481189136Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:50:03.488585709Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:50:33.514151052Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:50:33.52166502Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:50:47.361452004Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:50:47.368389296Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:51:03.545090041Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:51:03.553981548Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:51:33.563359491Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:51:33.570025084Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:51:47.386614483Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:51:47.396892247Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:52:03.588895346Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:52:03.594877653Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:52:33.616245554Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:52:33.624962392Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:52:47.417113257Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:52:47.42368318Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:53:03.646163976Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:53:03.652336677Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:53:33.674370843Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:53:33.680194744Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:53:47.440542733Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:53:47.447679205Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:54:03.6952896Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:54:03.700861433Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:54:33.722148913Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:54:33.72749653Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:54:47.463275342Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:54:47.469674923Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:55:03.745698741Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:55:03.751212802Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:55:33.77143665Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:55:33.777238747Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:55:47.486716668Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:55:47.497164328Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:56:03.800190232Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:56:03.80929164Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:56:33.823558193Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:56:33.829054288Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:56:47.506975165Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:56:47.514947011Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:57:03.848992346Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:57:03.854841751Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:57:33.879603428Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:57:33.888296317Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:57:47.525524747Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:57:47.532144492Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:58:03.913872246Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:58:03.922985539Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:58:33.945523221Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:58:33.951737081Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:58:47.547022379Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:58:47.558208131Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:59:03.973569449Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:59:03.980823712Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:59:34.001739408Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T13:59:34.009326882Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T13:59:47.57507789Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T13:59:47.581962395Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T14:00:04.030594366Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T14:00:04.037738722Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T14:00:34.05933959Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T14:00:34.065539035Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T14:00:47.60780963Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T14:00:47.620281305Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T14:01:04.086247244Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T14:01:04.092084895Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T14:01:34.114887383Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T14:01:34.122162179Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T14:01:47.632038412Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T14:01:47.639481613Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T14:02:04.1434311Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T14:02:04.149429269Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T14:02:34.17213999Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T14:02:34.178675741Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T14:02:47.658799972Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T14:02:47.665848019Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T14:03:04.201370455Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T14:03:04.20790816Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T14:03:34.228562107Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T14:03:34.234026739Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T14:03:47.67759513Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T14:03:47.685264644Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T14:04:04.255061567Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T14:04:04.261342478Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T14:04:34.270675066Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T14:04:34.275914893Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T14:04:47.700304821Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T14:04:47.707529343Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T14:05:04.298374366Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T14:05:04.303820965Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T14:05:34.325501893Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T14:05:34.330739304Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T14:05:47.718205932Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T14:05:47.726033315Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T14:06:04.350012505Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T14:06:04.355374427Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T14:06:34.376163119Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T14:06:34.381964992Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T14:06:47.742018742Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T14:06:47.749182932Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T14:07:04.401246319Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T14:07:04.40696121Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T14:07:34.420092927Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T14:07:34.425546624Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T14:07:47.759033967Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T14:07:47.766538687Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T14:08:04.442854176Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T14:08:04.448107472Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T14:08:34.468230485Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T14:08:34.473649686Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T14:08:47.80777361Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T14:08:47.831472505Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T14:09:04.485958369Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T14:09:04.491957462Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T14:09:34.511843146Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T14:09:34.518661943Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T14:09:47.854993639Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T14:09:47.862162483Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T14:10:04.537430018Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T14:10:04.562517809Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T14:10:34.584199014Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T14:10:34.589907311Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T14:10:47.883352573Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T14:10:47.891851425Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T14:11:04.614248903Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T14:11:04.619564034Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T14:11:34.643557744Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T14:11:34.650739567Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"
level=warn caller=logger.go:55 ts=2024-07-12T14:11:47.904506663Z component=forwarder/worker msg="Failed to retrieve metrics" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T14:11:47.912842445Z component=forwarder/worker msg="unable to forward results" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=%7B__name__%3D%22%3Anode_memory_MemAvailable_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_labels%22%7D&match%5B%5D=%7B__name__%3D%22authenticated_user_requests%22%7D&match%5B%5D=%7B__name__%3D%22authentication_attempts%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_cpu_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acontainer_spec_cpu_shares%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Ausage%3Aresources%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22container_spec_cpu_quota%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22coredns_dns_request_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22coredns_forward_responses_total%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_mvcc_db_total_size_in_bytes%22%7D&match%5B%5D=%7B__name__%3D%22etcd_debugging_snap_save_total_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_backend_commit_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_bucket%22%7D&match%5B%5D=%7B__name__%3D%22etcd_disk_wal_fsync_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22etcd_object_counts%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_client_grpc_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_received_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_network_peer_sent_bytes_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_client_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_has_leader%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_health_failures%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_leader_changes_seen_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_failed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_pending%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_committed_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_proposals_applied_total%22%7D&match%5B%5D=%7B__name__%3D%22etcd_server_quota_backend_bytes%22%7D&match%5B%5D=%7B__name__%3D%22grpc_server_started_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connection_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_connections_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_current_queue%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_http_average_response_latency_milliseconds%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_max_sessions%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_response_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22haproxy_backend_up%22%7D&match%5B%5D=%7B__name__%3D%22http_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_filesystem_usage%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_cpu_utilisation%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_load1_per_cpu%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_memory_utilisation%3Aratio%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_receive_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_bytes_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_network_transmit_drop_excluding_lo%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_num_cpu%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Anode_vmstat_pgmajfault%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22instance_device%3Anode_disk_io_time_weighted_seconds%3Arate1m%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_desired_number_scheduled%22%7D&match%5B%5D=%7B__name__%3D%22kube_daemonset_status_number_unavailable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_spec_unschedulable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_allocatable_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_pods%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_capacity_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_node_status_condition%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_limits_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_container_resource_requests_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_info%22%7D&match%5B%5D=%7B__name__%3D%22kube_pod_owner%22%7D&match%5B%5D=%7B__name__%3D%22kube_resourcequota%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_running_container_count%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_runtime_operations_duration_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_available_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kubelet_volume_stats_capacity_bytes%22%7D&match%5B%5D=%7B__name__%3D%22kube_persistentvolume_status_phase%22%7D&match%5B%5D=%7B__name__%3D%22machine_cpu_cores%22%7D&match%5B%5D=%7B__name__%3D%22machine_memory_bytes%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_low_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_medium_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_high_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_average_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_total_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_control_planes_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_available_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_deleted_hosted_clusters_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hypershift_operator_degraded_bool%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_hosted_control_planes_status_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_based_hcp_capacity_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_worker_node_resource_capacities_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_qps_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mce_hs_addon_request_based_hcp_capacity_current_gauge%22%7D&match%5B%5D=%7B__name__%3D%22mixin_pod_workload%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Akube_pod_container_resource_requests_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_memory%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Acontainer_memory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_cpu%3Akube_pod_container_resource_requests%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_cpu_seconds_total%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_avail_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_free_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_filesystem_size_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_memory_MemAvailable_bytes%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_OutSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_Tcp_RetransSegs%22%7D&match%5B%5D=%7B__name__%3D%22node_netstat_TcpExt_TCPSynRetrans%22%7D&match%5B%5D=%7B__name__%3D%22policyreport_info%22%7D&match%5B%5D=%7B__name__%3D%22up%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_errors_total%22%7D&match%5B%5D=%7B__name__%3D%22prometheus_operator_reconcile_operations_total%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_count%22%7D&match%5B%5D=%7B__name__%3D%22policy%3Apolicy_governance_info%3Apropagated_noncompliant_count%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hyperconverged_operator_health_status%22%7D&match%5B%5D=%7B__name__%3D%22kubevirt_hco_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_queue_duration_seconds_bucket%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_adds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22workqueue_depth%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22go_goroutines%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_cpu_seconds_total%22%2Cjob%3D%22apiserver%22%7D&match%5B%5D=%7B__name__%3D%22process_resident_memory_bytes%22%2Cjob%3D%22apiserver%22%7D\": tls: failed to verify certificate: x509: certificate signed by unknown authority"
level=error caller=logger.go:60 ts=2024-07-12T14:12:04.67259042Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+avg%28rate%28node_cpu_seconds_total%7Bmode%3D%22idle%22%7D%5B5m%5D%29%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70"
level=error caller=logger.go:60 ts=2024-07-12T14:12:04.677718774Z component=collectrule/evaluator msg="failed to evaluate collect rule" err="Get \"https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=%281+-+sum%28%3Anode_memory_MemAvailable_bytes%3Asum%29+%2F+sum%28kube_node_status_allocatable%7Bresource%3D%22memory%22%7D%29%29+%2A+100+%3E+70\": tls: failed to verify certificate: x509: certificate signed by unknown authority" rule="(1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70"